LLVM: lib/Transforms/Coroutines/CoroSplit.cpp Source File (original) (raw)

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

71#include

72#include

73#include

74#include <initializer_list>

75#include

76

77using namespace llvm;

78

79#define DEBUG_TYPE "coro-split"

80

81namespace {

82

83

84

85

86

88 TimeTraceScope FunctionScope("CollectCommonDebugInfo");

89

92 F, CloneFunctionChangeType::LocalChangesOnly, DIFinder);

93

95 DIFinder, SPClonedWithinModule);

96}

97}

98

99

100

101

102

108

110

112

113

114

117

118 if (auto Invoke = dyn_cast(CB)) {

119 auto WrapperInvoke =

121 Invoke->getUnwindDest(), {Awaiter, FramePtr});

122

123 WrapperInvoke->setCallingConv(Invoke->getCallingConv());

124 std::copy(Invoke->bundle_op_info_begin(), Invoke->bundle_op_info_end(),

125 WrapperInvoke->bundle_op_info_begin());

126 WrapperInvoke->setAttributes(NewAttributes);

127 WrapperInvoke->setDebugLoc(Invoke->getDebugLoc());

128 NewCall = WrapperInvoke;

129 } else if (auto Call = dyn_cast(CB)) {

131

133 WrapperCall->setDebugLoc(Call->getDebugLoc());

134 NewCall = WrapperCall;

135 } else {

136 llvm_unreachable("Unexpected coro_await_suspend invocation method");

137 }

138

140 Intrinsic::coro_await_suspend_handle) {

141

142

143 if (auto *Invoke = dyn_cast(CB)) {

144

145 Builder.SetInsertPoint(Invoke->getNormalDest()->getFirstInsertionPt());

146 }

147

151

154 Type::getVoidTy(Ctx), PointerType::getUnqual(Ctx), false);

155 auto *ResumeCall = Builder.CreateCall(ResumeTy, ResumeAddr, {NewCall});

157

158

159

161

162 NewCall = ResumeCall;

163 }

164

167}

168

173}

174

178 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce);

180 return;

181

183}

184

185

186

187

190

191 auto *EndAsync = dyn_cast(End);

192 if (!EndAsync) {

194 return true ;

195 }

196

197 auto *MustTailCallFunc = EndAsync->getMustTailCallFunction();

198 if (!MustTailCallFunc) {

200 return true ;

201 }

202

203

204 auto *CoroEndBlock = End->getParent();

205 auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor();

206 assert(MustTailCallFuncBlock && "Must have a single predecessor block");

207 auto It = MustTailCallFuncBlock->getTerminator()->getIterator();

208 auto *MustTailCall = cast(&*std::prev(It));

209 CoroEndBlock->splice(End->getIterator(), MustTailCallFuncBlock,

210 MustTailCall->getIterator());

211

212

216

217

218 auto *BB = End->getParent();

219 BB->splitBasicBlock(End);

220 BB->getTerminator()->eraseFromParent();

221

222 auto InlineRes = InlineFunction(*MustTailCall, FnInfo);

223 assert(InlineRes.isSuccess() && "Expected inlining to succeed");

224 (void)InlineRes;

225

226

227 return false;

228}

229

230

234

236

237

238 switch (Shape.ABI) {

239

240 case coro::ABI::Switch:

241 assert(!cast(End)->hasResults() &&

242 "switch coroutine should not return any values");

243

244

245 if (!InResume)

246 return;

248 break;

249

250

251 case coro::ABI::Async: {

253 if (!CoroEndBlockNeedsCleanup)

254 return;

255 break;

256 }

257

258

259

260 case coro::ABI::RetconOnce: {

262 auto *CoroEnd = cast(End);

264

265 if (!CoroEnd->hasResults()) {

268 break;

269 }

270

271 auto *CoroResults = CoroEnd->getResults();

272 unsigned NumReturns = CoroResults->numReturns();

273

274 if (auto *RetStructTy = dyn_cast(RetTy)) {

275 assert(RetStructTy->getNumElements() == NumReturns &&

276 "numbers of returns should match resume function singature");

278 unsigned Idx = 0;

279 for (Value *RetValEl : CoroResults->return_values())

282 } else if (NumReturns == 0) {

285 } else {

286 assert(NumReturns == 1);

287 Builder.CreateRet(*CoroResults->retval_begin());

288 }

291 CoroResults->eraseFromParent();

292 break;

293 }

294

295

296

297 case coro::ABI::Retcon: {

298 assert(!cast(End)->hasResults() &&

299 "retcon coroutine should not return any values");

302 auto RetStructTy = dyn_cast(RetTy);

304 cast(RetStructTy ? RetStructTy->getElementType(0) : RetTy);

305

307 if (RetStructTy) {

309 ReturnValue, 0);

310 }

312 break;

313 }

314 }

315

316

317 auto *BB = End->getParent();

318 BB->splitBasicBlock(End);

319 BB->getTerminator()->eraseFromParent();

320}

321

322

323

324

325

326

327

328

329

333 Shape.ABI == coro::ABI::Switch &&

334 "markCoroutineAsDone is only supported for Switch-Resumed ABI for now.");

337 "ResumeFn.addr");

341

342

343

344

345

346

347

348

349

352 assert(cast(Shape.CoroSuspends.back())->isFinal() &&

353 "The final suspend should only live in the last position of "

354 "CoroSuspends.");

358

359 Builder.CreateStore(IndexVal, FinalIndex);

360 }

361}

362

363

368

369 switch (Shape.ABI) {

370

371 case coro::ABI::Switch: {

372

373

374

375

376

377

379 if (!InResume)

380 return;

381 break;

382 }

383

384 case coro::ABI::Async:

385 break;

386

387 case coro::ABI::Retcon:

388 case coro::ABI::RetconOnce:

390 break;

391 }

392

393

395 auto *FromPad = cast(Bundle->Inputs[0]);

396 auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);

397 End->getParent()->splitBasicBlock(End);

398 CleanupRet->getParent()->getTerminator()->eraseFromParent();

399 }

400}

401

404 if (End->isUnwind())

406 else

408

409 auto &Context = End->getContext();

412 End->eraseFromParent();

413}

414

415

416

417

418

419

420

421

422

423

427

429 return;

430

432 auto FinalCaseIt = std::prev(Switch->case_end());

433 BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor();

434 Switch->removeCase(FinalCaseIt);

439

441

442

444 } else {

447 "ResumeFn.addr");

448 auto *Load =

452 }

454 }

455}

456

459 auto *AsyncSuspend = cast(Suspend);

460 auto *StructTy = cast(AsyncSuspend->getType());

461 auto &Context = Suspend->getParent()->getParent()->getContext();

464}

465

467 const Twine &Suffix,

474

477 OrigF.getName() + Suffix);

478

479 M->getFunctionList().insert(InsertBefore, NewF);

480

481 return NewF;

482}

483

484

485

486

487

491

492 auto NewS = VMap[ActiveSuspend];

493 if (NewS->use_empty())

494 return;

495

496

497

499

501 for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()),

502 E = NewF->arg_end();

503 I != E; ++I)

504 Args.push_back(&*I);

505

506

507

508 if (!isa(NewS->getType())) {

509 assert(Args.size() == 1);

510 NewS->replaceAllUsesWith(Args.front());

511 return;

512 }

513

514

516 auto *EVI = dyn_cast(U.getUser());

517 if (!EVI || EVI->getNumIndices() != 1)

518 continue;

519

520 EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]);

521 EVI->eraseFromParent();

522 }

523

524

525 if (NewS->use_empty())

526 return;

527

528

531 Aggr = Builder.CreateInsertValue(Aggr, Arg, Idx);

532

534}

535

537 Value *SuspendResult;

538

540

541

542

543

544

545

547 SuspendResult = Builder.getInt8(isSwitchDestroyFunction() ? 1 : 0);

548 break;

549

550

552 return;

553

554

555

556

559 return;

560 }

561

563

564 if (CS == ActiveSuspend)

565 continue;

566

567 auto *MappedCS = cast(VMap[CS]);

568 MappedCS->replaceAllUsesWith(SuspendResult);

569 MappedCS->eraseFromParent();

570 }

571}

572

575

576

577 auto *NewCE = cast(VMap[CE]);

579 }

580}

581

585 return;

586 Value *CachedSlot = nullptr;

587 auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * {

588 if (CachedSlot)

589 return CachedSlot;

590

591

592 for (auto &Arg : F.args()) {

593 if (Arg.isSwiftError()) {

594 CachedSlot = &Arg;

595 return &Arg;

596 }

597 }

598

599

601 F.getEntryBlock().getFirstNonPHIOrDbg());

604

605 CachedSlot = Alloca;

606 return Alloca;

607 };

608

610 auto MappedOp = VMap ? cast((*VMap)[Op]) : Op;

612

613

614 Value *MappedResult;

615 if (Op->arg_empty()) {

616 auto ValueTy = Op->getType();

617 auto Slot = getSwiftErrorSlot(ValueTy);

618 MappedResult = Builder.CreateLoad(ValueTy, Slot);

619 } else {

621 auto Value = MappedOp->getArgOperand(0);

623 auto Slot = getSwiftErrorSlot(ValueTy);

625 MappedResult = Slot;

626 }

627

628 MappedOp->replaceAllUsesWith(MappedResult);

629 MappedOp->eraseFromParent();

630 }

631

632

633 if (VMap == nullptr) {

635 }

636}

637

638

639static std::pair<SmallVector<DbgVariableIntrinsic *, 8>,

646 DbgVariableRecords.push_back(&DVR);

647 if (auto *DVI = dyn_cast(&I))

649 }

650 return {Intrinsics, DbgVariableRecords};

651}

652

655}

656

660

661

662 bool UseEntryValue =

668

669

670

672 auto IsUnreachableBlock = [&](BasicBlock *BB) {

674 &DomTree);

675 };

676 auto RemoveOne = [&](auto *DVI) {

677 if (IsUnreachableBlock(DVI->getParent()))

678 DVI->eraseFromParent();

679 else if (isa_and_nonnull(DVI->getVariableLocationOp(0))) {

680

681 unsigned Uses = 0;

682 for (auto *User : DVI->getVariableLocationOp(0)->users())

683 if (auto *I = dyn_cast(User))

684 if (!isa(I) && !IsUnreachableBlock(I->getParent()))

687 DVI->eraseFromParent();

688 }

689 };

690 for_each(Worklist, RemoveOne);

691 for_each(DbgVariableRecords, RemoveOne);

692}

693

695

696

697

698

699

701 auto *OldEntry = &NewF->getEntryBlock();

702 Entry->setName("entry" + Suffix);

703 Entry->moveBefore(OldEntry);

704 Entry->getTerminator()->eraseFromParent();

705

706

707

708

709 assert(Entry->hasOneUse());

710 auto BranchToEntry = cast(Entry->user_back());

711 assert(BranchToEntry->isUnconditional());

712 Builder.SetInsertPoint(BranchToEntry);

713 Builder.CreateUnreachable();

714 BranchToEntry->eraseFromParent();

715

716

717 Builder.SetInsertPoint(Entry);

720

721

722 auto *SwitchBB =

724 Builder.CreateBr(SwitchBB);

725 break;

726 }

730

731

732

734 isa(ActiveSuspend)) ||

737 isa(ActiveSuspend)));

738 auto *MappedCS = cast(VMap[ActiveSuspend]);

739 auto Branch = cast(MappedCS->getNextNode());

740 assert(Branch->isUnconditional());

741 Builder.CreateBr(Branch->getSuccessor(0));

742 break;

743 }

744 }

745

746

747

748 Function *F = OldEntry->getParent();

751 auto *Alloca = dyn_cast(&I);

752 if (!Alloca || I.use_empty())

753 continue;

755 !isa(Alloca->getArraySize()))

756 continue;

757 I.moveBefore(*Entry, Entry->getFirstInsertionPt());

758 }

759}

760

761

763

764

766

768 return &*NewF->arg_begin();

769

770

771

772

773

775 auto *ActiveAsyncSuspend = cast(ActiveSuspend);

776 auto ContextIdx = ActiveAsyncSuspend->getStorageArgumentIndex() & 0xff;

777 auto *CalleeContext = NewF->getArg(ContextIdx);

778 auto *ProjectionFunc =

779 ActiveAsyncSuspend->getAsyncContextProjectionFunction();

780 auto DbgLoc =

781 cast(VMap[ActiveSuspend])->getDebugLoc();

782

783 auto *CallerContext = Builder.CreateCall(ProjectionFunc->getFunctionType(),

784 ProjectionFunc, CalleeContext);

785 CallerContext->setCallingConv(ProjectionFunc->getCallingConv());

786 CallerContext->setDebugLoc(DbgLoc);

787

788 auto &Context = Builder.getContext();

789 auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(

792

795 assert(InlineRes.isSuccess());

796 (void)InlineRes;

797 return FramePtrAddr;

798 }

799

802 Argument *NewStorage = &*NewF->arg_begin();

804

805

807 return NewStorage;

808

809

810 return Builder.CreateLoad(FramePtrTy, NewStorage);

811 }

812 }

814}

815

816

817

818

819

820

821

824 if (!ActiveSuspend)

825 return;

826

828

829

830 if (auto *Branch = dyn_cast_or_null(Successor);

831 Branch && Branch->isUnconditional())

832 Successor = &*Branch->getSuccessor(0)->getFirstNonPHIOrDbg();

833

834

835

838 if (DL || DL.getLine() == 0)

839 continue;

840

841 if (SPToUpdate.getFile() == DL->getFile()) {

842 SPToUpdate.setScopeLine(DL.getLine());

843 return;

844 }

845

846 break;

847 }

848

849

851 if (SPToUpdate.getFile() == DL->getFile())

852 SPToUpdate.setScopeLine(DL->getLine());

853}

854

857 Align Alignment, bool NoAlias) {

861

862 if (NoAlias)

864

867 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);

868}

869

871 unsigned ParamIndex) {

873 ParamAttrs.addAttribute(Attribute::SwiftAsync);

874 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);

875}

876

878 unsigned ParamIndex) {

880 ParamAttrs.addAttribute(Attribute::SwiftSelf);

881 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);

882}

883

884

885

888

889

890

891

892

893

895 for (Argument &A : OrigF.args()) {

897 VMap[&A] = DummyArgs.back();

898 }

899

901

902

903

904 auto savedVisibility = NewF->getVisibility();

905 auto savedUnnamedAddr = NewF->getUnnamedAddr();

906 auto savedDLLStorageClass = NewF->getDLLStorageClass();

907

908

909

910

911 auto savedLinkage = NewF->getLinkage();

913

916 &CommonDebugInfo);

918 nullptr, nullptr, &CommonDebugInfo);

919

920 auto &Context = NewF->getContext();

921

923 assert(SP != OrigF.getSubprogram() && SP->isDistinct());

925

926

927

928

929

930

931

932

933

934 if (SP->getUnit() &&

935 SP->getUnit()->getSourceLanguage() == dwarf::DW_LANG_Swift) {

936 SP->replaceLinkageName(MDString::get(Context, NewF->getName()));

937 if (auto *Decl = SP->getDeclaration()) {

939 Decl->getContext(), Decl->getScope(), Decl->getName(),

940 NewF->getName(), Decl->getFile(), Decl->getLine(), Decl->getType(),

941 Decl->getScopeLine(), Decl->getContainingType(),

942 Decl->getVirtualIndex(), Decl->getThisAdjustment(),

943 Decl->getFlags(), Decl->getSPFlags(), Decl->getUnit(),

944 Decl->getTemplateParams(), nullptr, Decl->getRetainedNodes(),

945 Decl->getThrownTypes(), Decl->getAnnotations(),

946 Decl->getTargetFuncName());

947 SP->replaceDeclaration(NewDecl);

948 }

949 }

950 }

951

952 NewF->setLinkage(savedLinkage);

953 NewF->setVisibility(savedVisibility);

954 NewF->setUnnamedAddr(savedUnnamedAddr);

955 NewF->setDLLStorageClass(savedDLLStorageClass);

956

957

958

960 NewF->hasMetadata(LLVMContext::MD_func_sanitize))

961 NewF->eraseMetadata(LLVMContext::MD_func_sanitize);

962

963

964 auto OrigAttrs = NewF->getAttributes();

966

969

970

971 NewAttrs = NewAttrs.addFnAttributes(

972 Context, AttrBuilder(Context, OrigAttrs.getFnAttrs()));

973

976 break;

978 auto *ActiveAsyncSuspend = cast(ActiveSuspend);

980 Attribute::SwiftAsync)) {

981 uint32_t ArgAttributeIndices =

982 ActiveAsyncSuspend->getStorageArgumentIndex();

983 auto ContextArgIndex = ArgAttributeIndices & 0xff;

985

986

987

988 auto SwiftSelfIndex = ArgAttributeIndices >> 8;

989 if (SwiftSelfIndex)

991 }

992

993

994 auto FnAttrs = OrigF.getAttributes().getFnAttrs();

995 NewAttrs = NewAttrs.addFnAttributes(Context, AttrBuilder(Context, FnAttrs));

996 break;

997 }

1000

1001

1003

1004

1008 true);

1009

1010 break;

1011 }

1012

1014

1015

1016

1017

1020

1023 break;

1024

1025

1026

1027

1029 break;

1030

1031

1032

1033

1035 break;

1036 }

1037

1038 NewF->setAttributes(NewAttrs);

1040

1041

1042 replaceEntryBlock();

1043

1044

1046 ResumeCall = cast(VMap[ResumeCall]);

1048

1049

1051 }

1052

1053

1054

1058 Builder.CreateRetVoid();

1060 }

1061

1062 Builder.SetInsertPoint(&NewF->getEntryBlock().front());

1063 NewFramePtr = deriveNewFramePointer();

1064

1065

1067 NewFramePtr->takeName(OldFramePtr);

1069

1070

1071 auto *NewVFrame = Builder.CreateBitCast(

1074 if (OldVFrame != NewVFrame)

1076

1077

1078

1079 for (Instruction *DummyArg : DummyArgs) {

1080 DummyArg->replaceAllUsesWith(PoisonValue::get(DummyArg->getType()));

1081 DummyArg->deleteValue();

1082 }

1083

1086

1087

1088

1090 handleFinalSuspend();

1091 break;

1095

1096

1097 assert(ActiveSuspend != nullptr &&

1098 "no active suspend when lowering a continuation-style coroutine");

1099 replaceRetconOrAsyncSuspendUses();

1100 break;

1101 }

1102

1103

1104 replaceCoroSuspends();

1105

1106

1108

1109

1110 replaceCoroEnds();

1111

1112

1114}

1115

1117

1119 ActiveSuspend);

1120

1121

1123

1124

1125

1128}

1129

1132

1133 auto *FuncPtrStruct = cast(

1135 auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0);

1136 auto *OrigContextSize = FuncPtrStruct->getOperand(1);

1137 auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(),

1140 FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize);

1141

1143}

1144

1146

1147 auto *SizeIntrin = Shape.CoroSizes.back();

1148 Module *M = SizeIntrin->getModule();

1150 return DL.getTypeAllocSize(Shape.FrameTy);

1151}

1152

1156

1161 }

1162

1164 return;

1165

1166

1167 auto *SizeIntrin = Shape.CoroSizes.back();

1168 auto *SizeConstant =

1170

1174 }

1175}

1176

1179

1180#ifndef NDEBUG

1181

1182

1183

1186#endif

1187}

1188

1189

1190

1192 auto *CoroBegin = Shape.CoroBegin;

1193 switch (Shape.ABI) {

1198 if (AllocInst) {

1202 AllocInst->replaceAllUsesWith(Builder.getFalse());

1203 AllocInst->eraseFromParent();

1204 CoroBegin->replaceAllUsesWith(Frame);

1205 } else {

1206 CoroBegin->replaceAllUsesWith(CoroBegin->getMem());

1207 }

1208

1209 break;

1210 }

1214 CoroBegin->replaceAllUsesWith(PoisonValue::get(CoroBegin->getType()));

1215 break;

1216 }

1217

1218 CoroBegin->eraseFromParent();

1220}

1221

1222

1223

1224

1227

1228 if (isa(I))

1229 continue;

1230

1231 if (isa(I))

1232 return true;

1233 }

1234 return false;

1235}

1236

1240

1241 Set.insert(SaveBB);

1243

1244

1245

1246

1247 while (!Worklist.empty()) {

1249 Set.insert(BB);

1251 if (!Set.contains(Pred))

1253 }

1254

1255

1256 Set.erase(SaveBB);

1257 Set.erase(ResDesBB);

1258

1259 for (auto *BB : Set)

1261 return true;

1262

1263 return false;

1264}

1265

1267 auto *SaveBB = Save->getParent();

1268 auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();

1271

1272 if (SaveBB == ResumeOrDestroyBB)

1274

1275

1277 return true;

1278

1279

1281 {ResumeOrDestroyBB->getFirstNonPHIIt(), ResumeOrDestroyIt}))

1282 return true;

1283

1284

1286 return true;

1287

1288 return false;

1289}

1290

1291

1292

1296 if (!Prev) {

1297 auto *Pred = Suspend->getParent()->getSinglePredecessor();

1298 if (!Pred)

1299 return false;

1300 Prev = Pred->getTerminator();

1301 }

1302

1303 CallBase *CB = dyn_cast(Prev);

1304 if (!CB)

1305 return false;

1306

1308

1309

1310 auto *SubFn = dyn_cast(Callee);

1311 if (!SubFn)

1312 return false;

1313

1314

1315 if (SubFn->getFrame() != CoroBegin)

1316 return false;

1317

1318

1319

1320

1323 return false;

1324

1325

1326

1329 Save->eraseFromParent();

1330

1331

1332 if (auto *Invoke = dyn_cast(CB)) {

1334 }

1335

1336

1339

1340

1341 if (CalledValue != SubFn && CalledValue->user_empty())

1342 if (auto *I = dyn_cast(CalledValue))

1343 I->eraseFromParent();

1344

1345

1346 if (SubFn->user_empty())

1347 SubFn->eraseFromParent();

1348

1349 return true;

1350}

1351

1352

1354

1356 return;

1357

1359 size_t I = 0, N = S.size();

1360 if (N == 0)

1361 return;

1362

1363 size_t ChangedFinalIndex = std::numeric_limits<size_t>::max();

1364 while (true) {

1365 auto SI = cast(S[I]);

1366

1367

1369 if (--N == I)

1370 break;

1371

1373

1374 if (cast(S[I])->isFinal()) {

1376 ChangedFinalIndex = I;

1377 }

1378

1379 continue;

1380 }

1381 if (++I == N)

1382 break;

1383 }

1384 S.resize(N);

1385

1386

1387

1388 if (ChangedFinalIndex < N) {

1389 assert(cast(S[ChangedFinalIndex])->isFinal());

1390 std::swap(S[ChangedFinalIndex], S.back());

1391 }

1392}

1393

1394namespace {

1395

1396struct SwitchCoroutineSplitter {

1400 assert(Shape.ABI == coro::ABI::Switch);

1401

1402 MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F)};

1403

1404

1405

1406

1407 createResumeEntryBlock(F, Shape);

1409 F, ".resume", Shape, coro::CloneKind::SwitchResume, TTI,

1410 CommonDebugInfo);

1412 F, ".destroy", Shape, coro::CloneKind::SwitchUnwind, TTI,

1413 CommonDebugInfo);

1415 F, ".cleanup", Shape, coro::CloneKind::SwitchCleanup, TTI,

1416 CommonDebugInfo);

1417

1421

1422

1423 updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);

1424

1429

1430

1431

1432

1433 setCoroInfo(F, Shape, Clones);

1434 }

1435

1436

1437

1438

1439

1440

1441

1442

1445 assert(Shape.ABI == coro::ABI::Switch);

1446 auto *OrigFnTy = F.getFunctionType();

1447 auto OldParams = OrigFnTy->params();

1448

1450 NewParams.reserve(OldParams.size() + 1);

1451 NewParams.append(OldParams.begin(), OldParams.end());

1453

1454 auto *NewFnTy = FunctionType::get(OrigFnTy->getReturnType(), NewParams,

1455 OrigFnTy->isVarArg());

1457 Function::Create(NewFnTy, F.getLinkage(), F.getName() + ".noalloc");

1458

1460 unsigned int Idx = 0;

1461 for (const auto &I : F.args()) {

1463 }

1464

1465

1466 auto FrameIdx = NoAllocF->arg_size() - 1;

1469 CloneFunctionChangeType::LocalChangesOnly, Returns);

1470

1472 auto *NewCoroBegin =

1473 cast_if_present(VMap[Shape.CoroBegin]);

1474 auto *NewCoroId = cast(NewCoroBegin->getId());

1477 NewCoroBegin->replaceAllUsesWith(NoAllocF->getArg(FrameIdx));

1478 NewCoroBegin->eraseFromParent();

1479 }

1480

1482 M->getFunctionList().insert(M->end(), NoAllocF);

1483

1486

1487

1490 false);

1491

1493

1495

1496

1497 setCoroInfo(F, Shape, Clones);

1498

1499

1500

1502 return NoAllocF;

1503 }

1504

1505private:

1506

1507

1510

1511

1512

1513

1514

1515

1516

1517

1518

1519

1522

1525 auto *FrameTy = Shape.FrameTy;

1526 auto *GepIndex = Builder.CreateStructGEP(

1528 auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");

1530 Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());

1532

1533 size_t SuspendIndex = 0;

1535 auto *S = cast(AnyS);

1537

1538

1539

1540

1541 auto *Save = S->getCoroSave();

1542 Builder.SetInsertPoint(Save);

1543 if (S->isFinal()) {

1544

1545

1547 } else {

1548 auto *GepIndex = Builder.CreateStructGEP(

1550 Builder.CreateStore(IndexVal, GepIndex);

1551 }

1552

1554 Save->eraseFromParent();

1555

1556

1557

1558

1559

1560

1561

1562

1563

1564

1565

1566

1567

1568

1569

1570

1571

1572

1573

1574

1575

1576

1577

1578

1579 auto *SuspendBB = S->getParent();

1580 auto *ResumeBB =

1581 SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex));

1582 auto *LandingBB = ResumeBB->splitBasicBlock(

1583 S->getNextNode(), ResumeBB->getName() + Twine(".landing"));

1584 Switch->addCase(IndexVal, ResumeBB);

1585

1586 cast(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);

1588 PN->insertBefore(LandingBB->begin());

1589 S->replaceAllUsesWith(PN);

1590 PN->addIncoming(Builder.getInt8(-1), SuspendBB);

1591 PN->addIncoming(S, ResumeBB);

1592

1593 ++SuspendIndex;

1594 }

1595

1596 Builder.SetInsertPoint(UnreachBB);

1597 Builder.CreateUnreachable();

1598

1600 }

1601

1602

1606

1607 auto *ResumeAddr = Builder.CreateStructGEP(

1609 "resume.addr");

1610 Builder.CreateStore(ResumeFn, ResumeAddr);

1611

1612 Value *DestroyOrCleanupFn = DestroyFn;

1613

1616

1617

1618 DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);

1619 }

1620

1621 auto *DestroyAddr = Builder.CreateStructGEP(

1623 "destroy.addr");

1624 Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);

1625 }

1626

1627

1628

1629

1630

1631

1632

1633

1634

1635

1636

1637

1638

1639

1642

1643

1648 auto *ArrTy = ArrayType::get(Part->getType(), Args.size());

1649

1651 auto *GV = new GlobalVariable(*M, ConstVal->getType(), true,

1652 GlobalVariable::PrivateLinkage, ConstVal,

1653 F.getName() + Twine(".resumers"));

1654

1655

1659 }

1660};

1661

1662}

1663

1667 auto &Context = Suspend->getParent()->getParent()->getContext();

1669

1673 ResumeIntrinsic->eraseFromParent();

1676}

1677

1678

1682 size_t ArgIdx = 0;

1683 for (auto *paramTy : FnTy->params()) {

1685 if (paramTy != FnArgs[ArgIdx]->getType())

1688 else

1689 CallArgs.push_back(FnArgs[ArgIdx]);

1690 ++ArgIdx;

1691 }

1692}

1693

1699

1700

1703

1704 auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs);

1705

1708 }

1709 TailCall->setDebugLoc(Loc);

1710 TailCall->setCallingConv(MustTailCallFn->getCallingConv());

1711 return TailCall;

1712}

1713

1719

1720

1721 F.removeFnAttr(Attribute::NoReturn);

1722 F.removeRetAttr(Attribute::NoAlias);

1723 F.removeRetAttr(Attribute::NonNull);

1724

1725 auto &Context = F.getContext();

1727

1730

1731 auto *FramePtr = Id->getStorage();

1735 "async.ctx.frameptr");

1736

1737

1738 {

1739

1743 }

1744

1745

1746 auto NextF = std::next(F.getIterator());

1747

1748

1751 auto *Suspend = cast(CS);

1752

1753

1754 auto ResumeNameSuffix = ".resume.";

1755 auto ProjectionFunctionName =

1756 Suspend->getAsyncContextProjectionFunction()->getName();

1757 bool UseSwiftMangling = false;

1758 if (ProjectionFunctionName == "__swift_async_resume_project_context") {

1759 ResumeNameSuffix = "TQ";

1760 UseSwiftMangling = true;

1761 } else if (ProjectionFunctionName == "__swift_async_resume_get_context") {

1762 ResumeNameSuffix = "TY";

1763 UseSwiftMangling = true;

1764 }

1767 UseSwiftMangling ? ResumeNameSuffix + Twine(Idx) + "_"

1768 : ResumeNameSuffix + Twine(Idx),

1769 NextF, Suspend);

1771

1772

1773

1774 auto *SuspendBB = Suspend->getParent();

1775 auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);

1776 auto *Branch = cast(SuspendBB->getTerminator());

1777

1778

1779 auto *ReturnBB =

1781 Branch->setSuccessor(0, ReturnBB);

1782

1784

1785

1786 auto *Fn = Suspend->getMustTailCallFunction();

1791 FnArgs, Builder);

1795

1796

1798 }

1799

1801

1802 MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F)};

1803

1805 auto *Suspend = CS;

1806 auto *Clone = Clones[Idx];

1807

1809 Suspend, TTI, CommonDebugInfo);

1810 }

1811}

1812

1818

1819

1820

1821 F.removeFnAttr(Attribute::NoReturn);

1822 F.removeRetAttr(Attribute::NoAlias);

1823 F.removeRetAttr(Attribute::NonNull);

1824

1825

1827 Value *RawFramePtr;

1829 RawFramePtr = Id->getStorage();

1830 } else {

1832

1833

1836

1837

1838

1839

1841 RawFramePtr =

1843

1844

1845 Builder.CreateStore(RawFramePtr, Id->getStorage());

1846 }

1847

1848

1849 {

1850

1854 }

1855

1856

1858 PHINode *ContinuationPhi = nullptr;

1860

1861

1862 auto NextF = std::next(F.getIterator());

1863

1864

1867 auto Suspend = cast(CS);

1868

1869

1873

1874

1875

1876 auto SuspendBB = Suspend->getParent();

1877 auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);

1878 auto Branch = cast(SuspendBB->getTerminator());

1879

1880

1881 if (!ReturnBB) {

1882

1883 ReturnBB =

1886

1888

1889

1890 ContinuationPhi =

1892

1893

1895

1896

1900

1901

1902 auto RetTy = F.getReturnType();

1903

1904

1905

1906

1907 auto CastedContinuationTy =

1908 (ReturnPHIs.empty() ? RetTy : RetTy->getStructElementType(0));

1909 auto *CastedContinuation =

1910 Builder.CreateBitCast(ContinuationPhi, CastedContinuationTy);

1911

1912 Value *RetV = CastedContinuation;

1913 if (!ReturnPHIs.empty()) {

1914 auto ValueIdx = 0;

1916 RetV = Builder.CreateInsertValue(RetV, CastedContinuation, ValueIdx++);

1917

1918 for (auto Phi : ReturnPHIs)

1920 }

1921

1923 }

1924

1925

1926 Branch->setSuccessor(0, ReturnBB);

1927 assert(ContinuationPhi);

1929 for (auto [Phi, VUse] :

1931 Phi->addIncoming(VUse, SuspendBB);

1932 }

1933

1935

1936 MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F)};

1937

1939 auto Suspend = CS;

1940 auto Clone = Clones[Idx];

1941

1943 Suspend, TTI, CommonDebugInfo);

1944 }

1945}

1946

1947namespace {

1950

1951public:

1952 PrettyStackTraceFunction(Function &F) : F(F) {}

1954 OS << "While splitting coroutine ";

1955 F.printAsOperand(OS, false, F.getParent());

1956 OS << "\n";

1957 }

1958};

1959}

1960

1961

1966 }

1967 } else {

1969 auto &Context = End->getContext();

1971 End->eraseFromParent();

1972 }

1973 }

1974}

1975

1977 for (auto *U : F.users()) {

1978 if (auto *CB = dyn_cast(U)) {

1979 auto *Caller = CB->getFunction();

1980 if (Caller && Caller->isPresplitCoroutine() &&

1981 CB->hasFnAttr(llvm::Attribute::CoroElideSafe))

1982 return true;

1983 }

1984 }

1985 return false;

1986}

1987

1991 SwitchCoroutineSplitter::split(F, Shape, Clones, TTI);

1992}

1993

1996 bool OptimizeFrame) {

1997 PrettyStackTraceFunction prettyStackTrace(F);

1998

1999 auto &Shape = ABI.Shape;

2001

2003

2005

2007 ABI.buildCoroutineFrame(OptimizeFrame);

2009

2010 bool isNoSuspendCoroutine = Shape.CoroSuspends.empty();

2011

2012 bool shouldCreateNoAllocVariant =

2015

2016

2017

2018 if (isNoSuspendCoroutine) {

2020 } else {

2021 ABI.splitCoroutine(F, Shape, Clones, TTI);

2022 }

2023

2024

2025

2027

2028

2029

2030

2033 for (auto *DDI : DbgInsts)

2037

2039

2040 if (shouldCreateNoAllocVariant)

2041 SwitchCoroutineSplitter::createNoAllocVariant(F, Shape, Clones);

2042}

2043

2049

2050 auto *CurrentSCC = &C;

2051 if (!Clones.empty()) {

2052 switch (Shape.ABI) {

2054

2055

2056 for (Function *Clone : Clones)

2058 break;

2062

2063

2064 if (!Clones.empty())

2066 break;

2067 }

2068

2069

2071 UR, FAM);

2072 }

2073

2074

2075

2078 AM, UR, FAM);

2079 return *CurrentSCC;

2080}

2081

2082

2085 auto CastFn = Prepare->getArgOperand(0);

2087

2088

2089

2090

2091

2092

2093

2095

2096 auto *Cast = dyn_cast(U.getUser());

2097 if (!Cast || Cast->getType() != Fn->getType())

2098 continue;

2099

2100

2101 Cast->replaceAllUsesWith(Fn);

2102 Cast->eraseFromParent();

2103 }

2104

2105

2106

2109

2110

2111 while (auto *Cast = dyn_cast(CastFn)) {

2112 if (!Cast->use_empty())

2113 break;

2114 CastFn = Cast->getOperand(0);

2115 Cast->eraseFromParent();

2116 }

2117}

2118

2121 bool Changed = false;

2123

2124 auto *Prepare = cast(P.getUser());

2126 Changed = true;

2127 }

2128

2129 return Changed;

2130}

2131

2135 auto *PrepareFn = M.getFunction(Name);

2136 if (PrepareFn && !PrepareFn->use_empty())

2138}

2139

2140static std::unique_ptrcoro::BaseABI

2142 std::function<bool(Instruction &)> IsMatCallback,

2146 if (CustomABI >= GenCustomABIs.size())

2147 llvm_unreachable("Custom ABI not found amoung those specified");

2148 return GenCustomABIs[CustomABI](F, S);

2149 }

2150

2151 switch (S.ABI) {

2153 return std::make_uniquecoro::SwitchABI(F, S, IsMatCallback);

2155 return std::make_uniquecoro::AsyncABI(F, S, IsMatCallback);

2157 return std::make_uniquecoro::AnyRetconABI(F, S, IsMatCallback);

2159 return std::make_uniquecoro::AnyRetconABI(F, S, IsMatCallback);

2160 }

2162}

2163

2165 : CreateAndInitABI([](Function &F, coro::Shape &S) {

2166 std::unique_ptrcoro::BaseABI ABI =

2168 ABI->init();

2169 return ABI;

2170 }),

2171 OptimizeFrame(OptimizeFrame) {}

2172

2175 : CreateAndInitABI([=](Function &F, coro::Shape &S) {

2176 std::unique_ptrcoro::BaseABI ABI =

2178 ABI->init();

2179 return ABI;

2180 }),

2181 OptimizeFrame(OptimizeFrame) {}

2182

2183

2184

2186 bool OptimizeFrame)

2187 : CreateAndInitABI([=](Function &F, coro::Shape &S) {

2188 std::unique_ptrcoro::BaseABI ABI =

2190 ABI->init();

2191 return ABI;

2192 }),

2193 OptimizeFrame(OptimizeFrame) {}

2194

2195

2196

2198 std::function<bool(Instruction &)> IsMatCallback,

2200 : CreateAndInitABI([=](Function &F, coro::Shape &S) {

2201 std::unique_ptrcoro::BaseABI ABI =

2202 CreateNewABI(F, S, IsMatCallback, GenCustomABIs);

2203 ABI->init();

2204 return ABI;

2205 }),

2206 OptimizeFrame(OptimizeFrame) {}

2207

2211

2212

2213

2214 Module &M = *C.begin()->getFunction().getParent();

2215 auto &FAM =

2217

2218

2222

2223

2226 if (N.getFunction().isPresplitCoroutine())

2228

2229 if (Coroutines.empty() && PrepareFns.empty())

2231

2232 auto *CurrentSCC = &C;

2233

2236 LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()

2237 << "\n");

2238

2239

2240

2241

2243

2246 continue;

2247

2248 F.setSplittedCoroutine();

2249

2250 std::unique_ptrcoro::BaseABI ABI = CreateAndInitABI(F, Shape);

2251

2256 *N, Shape, Clones, *CurrentSCC, CG, AM, UR, FAM);

2257

2259 ORE.emit([&]() {

2261 << "Split '" << ore::NV("function", F.getName())

2262 << "' (frame_size=" << ore::NV("frame_size", Shape.FrameSize)

2264 });

2265

2267

2268 UR.CWorklist.insert(CurrentSCC);

2269 for (Function *Clone : Clones)

2271 }

2272 }

2273

2274 for (auto *PrepareFn : PrepareFns) {

2276 }

2277

2279}

amdgpu aa AMDGPU Address space based Alias Analysis Wrapper

AMDGPU Lower Kernel Arguments

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)

Expand Atomic instructions

This file contains the simple types necessary to represent the attributes associated with functions a...

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

This file provides interfaces used to manipulate a call graph, regardless if it is a "old style" Call...

This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...

This file contains the declarations for the subclasses of Constant, which represent the different fla...

static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)

static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy)

static std::pair< SmallVector< DbgVariableIntrinsic *, 8 >, SmallVector< DbgVariableRecord * > > collectDbgVariableIntrinsics(Function &F)

Returns all DbgVariableIntrinsic in F.

static LazyCallGraph::SCC & updateCallGraphAfterCoroutineSplit(LazyCallGraph::Node &N, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, LazyCallGraph::SCC &C, LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)

static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)

static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)

static void maybeFreeRetconStorage(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG)

static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB)

static Function * createCloneDeclaration(Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)

Remove calls to llvm coro end in the original static function void removeCoroEndsFromRampFunction(const coro::Shape &Shape)

static FunctionType * getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend)

static void updateScopeLine(Instruction *ActiveSuspend, DISubprogram &SPToUpdate)

Adjust the scope line of the funclet to the first line number after the suspend point.

static void addPrepareFunction(const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)

static void simplifySuspendPoints(coro::Shape &Shape)

static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment, bool NoAlias)

static bool hasSafeElideCaller(Function &F)

static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)

static void replaceFallthroughCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)

Replace a non-unwind call to llvm.coro.end.

static void replaceFrameSizeAndAlignment(coro::Shape &Shape)

static std::unique_ptr< coro::BaseABI > CreateNewABI(Function &F, coro::Shape &S, std::function< bool(Instruction &)> IsMatCallback, const SmallVector< CoroSplitPass::BaseABITy > GenCustomABIs)

static bool replaceCoroEndAsync(AnyCoroEndInst *End)

Replace an llvm.coro.end.async.

static void doSplitCoroutine(Function &F, SmallVectorImpl< Function * > &Clones, coro::BaseABI &ABI, TargetTransformInfo &TTI, bool OptimizeFrame)

static bool hasCallsInBlockBetween(iterator_range< BasicBlock::iterator > R)

Replace a call to llvm coro prepare static retcon void replacePrepare(CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)

static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)

Replace an unwind call to llvm.coro.end.

static bool simplifySuspendPoint(CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)

static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr)

static void updateAsyncFuncPointerContextSize(coro::Shape &Shape)

static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)

static void lowerAwaitSuspend(IRBuilder<> &Builder, CoroAwaitSuspendInst *CB, coro::Shape &Shape)

static void lowerAwaitSuspends(Function &F, coro::Shape &Shape)

static void handleNoSuspendCoroutine(coro::Shape &Shape)

static void postSplitCleanup(Function &F)

static TypeSize getFrameSizeForShape(coro::Shape &Shape)

Coerce the arguments in p FnArgs according to p FnTy in p static CallArgs void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > &CallArgs)

static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend, Value *Continuation)

Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx

This file defines the DenseMap class.

This file contains constants used for implementing Dwarf debug support.

This file provides various utilities for inspecting and working with the control flow graph in LLVM I...

Module.h This file contains the declarations for the Module class.

Implements a lazy call graph analysis and related passes for the new pass manager.

FunctionAnalysisManager FAM

This file provides a priority worklist.

const SmallVectorImpl< MachineOperand > & Cond

Remove Loads Into Fake Uses

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This file defines the SmallPtrSet class.

This file defines the SmallVector class.

static SymbolRef::Type getType(const Symbol *Sym)

This pass exposes codegen information to IR-level passes.

static const unsigned FramePtr

void setSwiftError(bool V)

Specify whether this alloca is used to represent a swifterror.

void setAlignment(Align Align)

A container for analyses that lazily runs them and caches their results.

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

CoroAllocInst * getCoroAlloc()

Align getStorageAlignment() const

uint64_t getStorageSize() const

This class represents an incoming formal argument to a Function.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

ArrayRef< T > drop_front(size_t N=1) const

Drop the first N elements of the array.

size_t size() const

size - Get the array size.

AttrBuilder & addAlignmentAttr(MaybeAlign Align)

This turns an alignment into the form used internally in Attribute.

AttrBuilder & addAttribute(Attribute::AttrKind Val)

Add an attribute to the builder.

AttrBuilder & addDereferenceableAttr(uint64_t Bytes)

This turns the number of dereferenceable bytes into the form used internally in Attribute.

AttributeList removeParamAttributes(LLVMContext &C, unsigned ArgNo, const AttributeMask &AttrsToRemove) const

Remove the specified attribute at the specified arg index from this attribute list.

LLVM Basic Block Representation.

static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)

Creates a new BasicBlock.

BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)

Split the basic block into two basic blocks at the specified instruction.

const Function * getParent() const

Return the enclosing method, or null if none.

InstListType::iterator iterator

Instruction iterators...

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

void setCallingConv(CallingConv::ID CC)

Function * getCalledFunction() const

Returns the function called, or null if this is an indirect function invocation or the function signa...

Value * getCalledOperand() const

void setAttributes(AttributeList A)

Set the attributes for this call.

Value * getArgOperand(unsigned i) const

AttributeList getAttributes() const

Return the attributes for this call.

The basic data container for the call graph of a Module of IR.

This class represents a function call, abstracting a target machine's calling convention.

static Constant * get(ArrayType *T, ArrayRef< Constant * > V)

static Constant * getPointerCast(Constant *C, Type *Ty)

Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.

This is the shared class of boolean and integer constants.

static ConstantInt * getTrue(LLVMContext &Context)

static ConstantInt * getFalse(LLVMContext &Context)

static ConstantPointerNull * get(PointerType *T)

Static factory methods - Return objects of the specified value.

static Constant * get(StructType *T, ArrayRef< Constant * > V)

static ConstantTokenNone * get(LLVMContext &Context)

Return the ConstantTokenNone.

This represents the llvm.coro.align instruction.

This represents the llvm.coro.alloc instruction.

This represents the llvm.coro.await.suspend.{void,bool,handle} instructions.

Value * getAwaiter() const

Function * getWrapperFunction() const

This class represents the llvm.coro.begin or llvm.coro.begin.custom.abi instructions.

AnyCoroIdInst * getId() const

bool hasCustomABI() const

This represents the llvm.coro.id instruction.

void setInfo(Constant *C)

This represents the llvm.coro.size instruction.

This represents the llvm.coro.suspend.async instruction.

CoroAsyncResumeInst * getResumeFunction() const

This represents the llvm.coro.suspend instruction.

CoroSaveInst * getCoroSave() const

DISubprogram * getSubprogram() const

Get the subprogram for this scope.

This class represents an Operation in the Expression.

A parsed version of the target data layout string in and methods for querying it.

This is the common base class for debug info intrinsics for variables.

Record of a variable value-assignment, aka a non instruction representation of the dbg....

Utility to find all debug info in a module.

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

bool isReachableFromEntry(const Use &U) const

Provide an overload for a Use.

This class represents a freeze function that returns random concrete value if an operand is either a ...

A proxy from a FunctionAnalysisManager to an SCC.

Class to represent function types.

Type * getReturnType() const

static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)

FunctionType * getFunctionType() const

Returns the FunctionType for me.

Intrinsic::ID getIntrinsicID() const LLVM_READONLY

getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...

CallingConv::ID getCallingConv() const

getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...

AttributeList getAttributes() const

Return the attribute list for this Function.

void setAttributes(AttributeList Attrs)

Set the attribute list for this Function.

LLVMContext & getContext() const

getContext - Return a reference to the LLVMContext associated with this function.

bool isCoroOnlyDestroyWhenComplete() const

Argument * getArg(unsigned i) const

void setLinkage(LinkageTypes LT)

Module * getParent()

Get the module that this global value is contained inside of...

PointerType * getType() const

Global values are always pointers.

@ InternalLinkage

Rename collisions when linking (static functions).

@ ExternalLinkage

Externally visible function.

const Constant * getInitializer() const

getInitializer - Return the initializer for this global variable.

void setInitializer(Constant *InitVal)

setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...

AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")

Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")

InvokeInst * CreateInvoke(FunctionType *Ty, Value *Callee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef< Value * > Args, ArrayRef< OperandBundleDef > OpBundles, const Twine &Name="")

Create an invoke instruction.

BasicBlock::iterator GetInsertPoint() const

Value * CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx, const Twine &Name="")

Value * CreateConstInBoundsGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")

CleanupReturnInst * CreateCleanupRet(CleanupPadInst *CleanupPad, BasicBlock *UnwindBB=nullptr)

ReturnInst * CreateRet(Value *V)

Create a 'ret ' instruction.

ConstantInt * getInt64(uint64_t C)

Get a constant 64-bit value.

Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")

PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")

Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")

BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)

Create a conditional 'br Cond, TrueDest, FalseDest' instruction.

LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)

Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...

LLVMContext & getContext() const

ReturnInst * CreateRetVoid()

Create a 'ret void' instruction.

StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)

ConstantInt * getFalse()

Get the constant value for i1 false.

CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)

BranchInst * CreateBr(BasicBlock *Dest)

Create an unconditional 'br label X' instruction.

Value * CreateIsNull(Value *Arg, const Twine &Name="")

Return a boolean value testing if Arg == 0.

void SetInsertPoint(BasicBlock *TheBB)

This specifies that created instructions should be appended to the end of the specified block.

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

This class captures the data input to the InlineFunction call, and records the auxiliary results prod...

const DebugLoc & getDebugLoc() const

Return the debug location for this node as a DebugLoc.

InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

const Instruction * getNextNonDebugInstruction(bool SkipPseudoOp=false) const

Return a pointer to the next non-debug instruction in the same basic block as 'this',...

This is an important class for using LLVM in a threaded context.

A node in the call graph.

An SCC of the call graph.

A lazily constructed view of the call graph of a module.

void addSplitFunction(Function &OriginalFunction, Function &NewFunction)

Add a new function split/outlined from an existing function.

void addSplitRefRecursiveFunctions(Function &OriginalFunction, ArrayRef< Function * > NewFunctions)

Add new ref-recursive functions split/outlined from an existing function.

Node & get(Function &F)

Get a graph node for a given function, scanning it to populate the graph data as necessary.

SCC * lookupSCC(Node &N) const

Lookup a function's SCC in the graph.

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

static MDString * get(LLVMContext &Context, StringRef Str)

A Module instance is used to store all the information related to an LLVM module.

FunctionListType::iterator iterator

The Function iterators.

void addIncoming(Value *V, BasicBlock *BB)

Add an incoming value to the end of the PHI list.

static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...

static PointerType * getUnqual(Type *ElementType)

This constructs a pointer to an object of the specified type in the default address space (address sp...

static PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

PrettyStackTraceEntry - This class is used to represent a frame of the "pretty" stack trace that is d...

Return a value (possibly void), from a function.

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void reserve(size_type N)

void append(ItTy in_start, ItTy in_end)

Add the specified range to the end of the SmallVector.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

Type * getTypeAtIndex(const Value *V) const

Given an index value into the type, return the type of the element.

Analysis pass providing the TargetTransformInfo.

This pass provides access to the codegen interfaces that are needed for IR-level transformations.

bool supportsTailCallFor(const CallBase *CB) const

If target supports tail call on CB.

The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.

Value handle that tracks a Value across RAUW.

ValueTy * getValPtr() const

Triple - Helper class for working with autoconf configuration names.

bool isArch64Bit() const

Test whether the architecture is 64-bit.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

static Type * getVoidTy(LLVMContext &C)

LLVMContext & getContext() const

Return the LLVMContext in which this type was uniqued.

static IntegerType * getInt8Ty(LLVMContext &C)

A Use represents the edge between a Value definition and its users.

void setOperand(unsigned i, Value *Val)

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

iterator_range< user_iterator > users()

const Value * stripPointerCasts() const

Strip off pointer casts, all-zero GEPs and address space casts.

LLVMContext & getContext() const

All values hold a context through their type.

iterator_range< use_iterator > uses()

StringRef getName() const

Return a constant reference to the value's name.

void takeName(Value *V)

Transfer the name from V to this value.

void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI) override

void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI) override

static Function * createClone(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, Function *NewF, AnyCoroSuspendInst *ActiveSuspend, TargetTransformInfo &TTI, const MetadataSetTy &CommonDebugInfo)

Create a clone for a continuation lowering.

void replaceSwiftErrorOps()

Value * deriveNewFramePointer()

Derive the value of the new frame pointer.

void replaceCoroSuspends()

void handleFinalSuspend()

bool isSwitchDestroyFunction()

void replaceRetconOrAsyncSuspendUses()

Replace uses of the active llvm.coro.suspend.retcon/async call with the arguments to the continuation...

virtual void create()

Clone the body of the original function into a resume function of some sort.

void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI) override

void create() override

Clone the body of the original function into a resume function of some sort.

static Function * createClone(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, CloneKind FKind, TargetTransformInfo &TTI, const MetadataSetTy &CommonDebugInfo)

Create a clone for a switch lowering.

const ParentTy * getParent() const

self_iterator getIterator()

A range adaptor for a pair of iterators.

This class implements an extremely fast bulk output stream that can only output to a stream.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

constexpr char Args[]

Key for Kernel::Metadata::mArgs.

@ Fast

Attempts to make calls as fast as possible (e.g.

@ C

The default llvm calling convention, compatible with C.

void salvageDebugInfo(SmallDenseMap< Argument *, AllocaInst *, 4 > &ArgToAllocaMap, DbgVariableIntrinsic &DVI, bool IsEntryPoint)

Attempts to rewrite the location operand of debug intrinsics in terms of the coroutine frame pointer,...

@ Async

The "async continuation" lowering, where each suspend point creates a single continuation function.

@ RetconOnce

The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...

@ Retcon

The "returned-continuation" lowering, where each suspend point creates a single continuation function...

@ Switch

The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...

void suppressCoroAllocs(CoroIdInst *CoroId)

Replaces all @llvm.coro.alloc intrinsics calls associated with a given call @llvm....

void normalizeCoroutine(Function &F, coro::Shape &Shape, TargetTransformInfo &TTI)

CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, TargetTransformInfo &TTI, ArrayRef< Value * > Arguments, IRBuilder<> &)

void replaceCoroFree(CoroIdInst *CoroId, bool Elide)

bool isTriviallyMaterializable(Instruction &I)

@ SwitchCleanup

The shared cleanup function for a switch lowering.

@ Continuation

An individual continuation function.

DiagnosticInfoOptimizationBase::Argument NV

This is an optimization pass for GlobalISel generic memory operations.

void CloneFunctionAttributesInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, bool ModuleLevelChanges, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)

Clone OldFunc's attributes into NewFunc, transforming values based on the mappings in VMap.

UnaryFunction for_each(R &&Range, UnaryFunction F)

Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.

detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)

zip iterator that assumes that all iteratees have the same length.

auto enumerate(FirstRange &&First, RestRanges &&...Rest)

Given two or more input ranges, returns a new range whose values are tuples (A, B,...

bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)

Check a function for errors, useful for use when debugging a pass.

LazyCallGraph::SCC & updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)

Helper to update the call graph after running a function pass.

LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)

Helper to update the call graph after running a CGSCC pass.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

void report_fatal_error(Error Err, bool gen_crash_diag=true)

Report a serious error, calling any installed error handler.

MetadataSetTy FindDebugInfoToIdentityMap(CloneFunctionChangeType Changes, DebugInfoFinder &DIFinder, DISubprogram *SPClonedWithinModule)

Based on Changes and DIFinder return debug info that needs to be identity mapped during Metadata clon...

unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)

Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...

raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

void CloneFunctionMetadataInto(Function &NewFunc, const Function &OldFunc, ValueToValueMapTy &VMap, RemapFlags RemapFlag, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr, const MetadataSetTy *IdentityMD=nullptr)

Clone OldFunc's metadata into NewFunc.

DWARFExpression::Operation Op

InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)

This function inlines the called function into the basic block of the caller.

void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)

Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.

auto predecessors(const MachineBasicBlock *BB)

DISubprogram * CollectDebugInfoForCloning(const Function &F, CloneFunctionChangeType Changes, DebugInfoFinder &DIFinder)

Collect debug information such as types, compile units, and other subprograms that are reachable from...

static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)

Filter the DbgRecord range to DbgVariableRecord types only and downcast.

bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)

Remove all blocks that can not be reached from the function's entry.

void CloneFunctionBodyInto(Function &NewFunc, const Function &OldFunc, ValueToValueMapTy &VMap, RemapFlags RemapFlag, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr, const MetadataSetTy *IdentityMD=nullptr)

Clone OldFunc's body into NewFunc.

bool isPotentiallyReachable(const Instruction *From, const Instruction *To, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet=nullptr, const DominatorTree *DT=nullptr, const LoopInfo *LI=nullptr)

Determine whether instruction 'To' is reachable from 'From', without passing through any blocks in Ex...

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.

This struct is a compact representation of a valid (non-zero power of two) alignment.

uint64_t value() const

This is a hole in the type system and should not be abused.

Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...

SmallPriorityWorklist< LazyCallGraph::SCC *, 1 > & CWorklist

Worklist of the SCCs queued for processing.

PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)

CoroSplitPass(bool OptimizeFrame=false)

BaseABITy CreateAndInitABI

CallInst * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)

GlobalVariable * AsyncFuncPointer

bool IsFrameInlineInStorage

Function * ResumePrototype

SwitchInst * ResumeSwitch

BasicBlock * ResumeEntryBlock

SmallVector< CallInst *, 2 > SymmetricTransfers

SmallVector< CoroAwaitSuspendInst *, 4 > CoroAwaitSuspends

AsyncLoweringStorage AsyncLowering

FunctionType * getResumeFunctionType() const

IntegerType * getIndexType() const

AnyCoroIdRetconInst * getRetconCoroId() const

PointerType * getSwitchResumePointerType() const

CoroIdInst * getSwitchCoroId() const

SmallVector< CoroSizeInst *, 2 > CoroSizes

CallingConv::ID getResumeFunctionCC() const

SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends

Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const

Allocate memory according to the rules of the active lowering.

ConstantInt * getIndex(uint64_t Value) const

SwitchLoweringStorage SwitchLowering

CoroBeginInst * CoroBegin

BasicBlock::iterator getInsertPtAfterFramePtr() const

ArrayRef< Type * > getRetconResultTypes() const

void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const

Deallocate memory according to the rules of the active lowering.

RetconLoweringStorage RetconLowering

SmallVector< CoroAlignInst *, 2 > CoroAligns

CoroIdAsyncInst * getAsyncCoroId() const

SmallVector< AnyCoroEndInst *, 4 > CoroEnds

SmallVector< CallInst *, 2 > SwiftErrorOps

BasicBlock * AllocaSpillBlock

unsigned getSwitchIndexField() const