CoroSplit.cpp Source File (original) (raw)

70#include

71#include

72#include

73#include <initializer_list>

74#include

76using namespace llvm;

78#define DEBUG_TYPE "coro-split"

99 if (auto Invoke = dyn_cast(CB)) {

100 auto WrapperInvoke =

102 Invoke->getUnwindDest(), {Awaiter, FramePtr});

103

104 WrapperInvoke->setCallingConv(Invoke->getCallingConv());

105 std::copy(Invoke->bundle_op_info_begin(), Invoke->bundle_op_info_end(),

106 WrapperInvoke->bundle_op_info_begin());

107 WrapperInvoke->setAttributes(NewAttributes);

108 WrapperInvoke->setDebugLoc(Invoke->getDebugLoc());

109 NewCall = WrapperInvoke;

110 } else if (auto Call = dyn_cast(CB)) {

112

114 WrapperCall->setDebugLoc(Call->getDebugLoc());

115 NewCall = WrapperCall;

116 } else {

117 llvm_unreachable("Unexpected coro_await_suspend invocation method");

118 }

119

121 Intrinsic::coro_await_suspend_handle) {

122

123

124 if (auto *Invoke = dyn_cast(CB)) {

125

126 Builder.SetInsertPoint(Invoke->getNormalDest()->getFirstInsertionPt());

127 }

128

132

135 Type::getVoidTy(Ctx), PointerType::getUnqual(Ctx), false);

136 auto *ResumeCall = Builder.CreateCall(ResumeTy, ResumeAddr, {NewCall});

138

139

140

142

143 NewCall = ResumeCall;

144 }

145

148}

149

154}

155

159 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce);

161 return;

162

164}

165

166

167

168

171

172 auto *EndAsync = dyn_cast(End);

173 if (!EndAsync) {

175 return true ;

176 }

177

178 auto *MustTailCallFunc = EndAsync->getMustTailCallFunction();

179 if (!MustTailCallFunc) {

181 return true ;

182 }

183

184

185 auto *CoroEndBlock = End->getParent();

186 auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor();

187 assert(MustTailCallFuncBlock && "Must have a single predecessor block");

188 auto It = MustTailCallFuncBlock->getTerminator()->getIterator();

189 auto *MustTailCall = cast(&*std::prev(It));

190 CoroEndBlock->splice(End->getIterator(), MustTailCallFuncBlock,

191 MustTailCall->getIterator());

192

193

197

198

199 auto *BB = End->getParent();

200 BB->splitBasicBlock(End);

201 BB->getTerminator()->eraseFromParent();

202

203 auto InlineRes = InlineFunction(*MustTailCall, FnInfo);

204 assert(InlineRes.isSuccess() && "Expected inlining to succeed");

205 (void)InlineRes;

206

207

208 return false;

209}

210

211

215

217

218

219 switch (Shape.ABI) {

220

221 case coro::ABI::Switch:

222 assert(!cast(End)->hasResults() &&

223 "switch coroutine should not return any values");

224

225

226 if (!InResume)

227 return;

229 break;

230

231

232 case coro::ABI::Async: {

234 if (!CoroEndBlockNeedsCleanup)

235 return;

236 break;

237 }

238

239

240

241 case coro::ABI::RetconOnce: {

243 auto *CoroEnd = cast(End);

245

246 if (!CoroEnd->hasResults()) {

249 break;

250 }

251

252 auto *CoroResults = CoroEnd->getResults();

253 unsigned NumReturns = CoroResults->numReturns();

254

255 if (auto *RetStructTy = dyn_cast(RetTy)) {

256 assert(RetStructTy->getNumElements() == NumReturns &&

257 "numbers of returns should match resume function singature");

259 unsigned Idx = 0;

260 for (Value *RetValEl : CoroResults->return_values())

263 } else if (NumReturns == 0) {

266 } else {

267 assert(NumReturns == 1);

268 Builder.CreateRet(*CoroResults->retval_begin());

269 }

272 CoroResults->eraseFromParent();

273 break;

274 }

275

276

277

278 case coro::ABI::Retcon: {

279 assert(!cast(End)->hasResults() &&

280 "retcon coroutine should not return any values");

283 auto RetStructTy = dyn_cast(RetTy);

285 cast(RetStructTy ? RetStructTy->getElementType(0) : RetTy);

286

288 if (RetStructTy) {

290 ReturnValue, 0);

291 }

293 break;

294 }

295 }

296

297

298 auto *BB = End->getParent();

299 BB->splitBasicBlock(End);

300 BB->getTerminator()->eraseFromParent();

301}

302

303

304

305

306

307

308

309

310

314 Shape.ABI == coro::ABI::Switch &&

315 "markCoroutineAsDone is only supported for Switch-Resumed ABI for now.");

318 "ResumeFn.addr");

322

323

324

325

326

327

328

329

330

333 assert(cast(Shape.CoroSuspends.back())->isFinal() &&

334 "The final suspend should only live in the last position of "

335 "CoroSuspends.");

339

340 Builder.CreateStore(IndexVal, FinalIndex);

341 }

342}

343

344

349

350 switch (Shape.ABI) {

351

352 case coro::ABI::Switch: {

353

354

355

356

357

358

360 if (!InResume)

361 return;

362 break;

363 }

364

365 case coro::ABI::Async:

366 break;

367

368 case coro::ABI::Retcon:

369 case coro::ABI::RetconOnce:

371 break;

372 }

373

374

376 auto *FromPad = cast(Bundle->Inputs[0]);

377 auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);

378 End->getParent()->splitBasicBlock(End);

379 CleanupRet->getParent()->getTerminator()->eraseFromParent();

380 }

381}

382

385 if (End->isUnwind())

387 else

389

390 auto &Context = End->getContext();

393 End->eraseFromParent();

394}

395

396

397

398

399

400

401

402

403

404

408

410 return;

411

413 auto FinalCaseIt = std::prev(Switch->case_end());

414 BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor();

415 Switch->removeCase(FinalCaseIt);

420

422

423

425 } else {

428 "ResumeFn.addr");

429 auto *Load =

433 }

435 }

436}

437

440 auto *AsyncSuspend = cast(Suspend);

441 auto *StructTy = cast(AsyncSuspend->getType());

442 auto &Context = Suspend->getParent()->getParent()->getContext();

445}

446

448 const Twine &Suffix,

455

458 OrigF.getName() + Suffix);

459

460 M->getFunctionList().insert(InsertBefore, NewF);

461

462 return NewF;

463}

464

465

466

467

468

472

473 auto NewS = VMap[ActiveSuspend];

474 if (NewS->use_empty())

475 return;

476

477

478

480

482 for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()),

483 E = NewF->arg_end();

484 I != E; ++I)

485 Args.push_back(&*I);

486

487

488

489 if (!isa(NewS->getType())) {

490 assert(Args.size() == 1);

491 NewS->replaceAllUsesWith(Args.front());

492 return;

493 }

494

495

497 auto *EVI = dyn_cast(U.getUser());

498 if (!EVI || EVI->getNumIndices() != 1)

499 continue;

500

501 EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]);

502 EVI->eraseFromParent();

503 }

504

505

506 if (NewS->use_empty())

507 return;

508

509

512 Aggr = Builder.CreateInsertValue(Aggr, Arg, Idx);

513

515}

516

518 Value *SuspendResult;

519

521

522

523

524

525

526

528 SuspendResult = Builder.getInt8(isSwitchDestroyFunction() ? 1 : 0);

529 break;

530

531

533 return;

534

535

536

537

540 return;

541 }

542

544

545 if (CS == ActiveSuspend)

546 continue;

547

548 auto *MappedCS = cast(VMap[CS]);

549 MappedCS->replaceAllUsesWith(SuspendResult);

550 MappedCS->eraseFromParent();

551 }

552}

553

556

557

558 auto *NewCE = cast(VMap[CE]);

560 }

561}

562

566 return;

567 Value *CachedSlot = nullptr;

568 auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * {

569 if (CachedSlot)

570 return CachedSlot;

571

572

573 for (auto &Arg : F.args()) {

574 if (Arg.isSwiftError()) {

575 CachedSlot = &Arg;

576 return &Arg;

577 }

578 }

579

580

581 IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());

584

585 CachedSlot = Alloca;

586 return Alloca;

587 };

588

590 auto MappedOp = VMap ? cast((*VMap)[Op]) : Op;

592

593

594 Value *MappedResult;

595 if (Op->arg_empty()) {

596 auto ValueTy = Op->getType();

597 auto Slot = getSwiftErrorSlot(ValueTy);

598 MappedResult = Builder.CreateLoad(ValueTy, Slot);

599 } else {

601 auto Value = MappedOp->getArgOperand(0);

603 auto Slot = getSwiftErrorSlot(ValueTy);

605 MappedResult = Slot;

606 }

607

608 MappedOp->replaceAllUsesWith(MappedResult);

609 MappedOp->eraseFromParent();

610 }

611

612

613 if (VMap == nullptr) {

615 }

616}

617

618

619static std::pair<SmallVector<DbgVariableIntrinsic *, 8>,

626 DbgVariableRecords.push_back(&DVR);

627 if (auto *DVI = dyn_cast(&I))

629 }

630 return {Intrinsics, DbgVariableRecords};

631}

632

635}

636

640

641

642 bool UseEntryValue =

648

649

650

652 auto IsUnreachableBlock = [&](BasicBlock *BB) {

654 &DomTree);

655 };

656 auto RemoveOne = [&](auto *DVI) {

657 if (IsUnreachableBlock(DVI->getParent()))

658 DVI->eraseFromParent();

659 else if (isa_and_nonnull(DVI->getVariableLocationOp(0))) {

660

661 unsigned Uses = 0;

662 for (auto *User : DVI->getVariableLocationOp(0)->users())

663 if (auto *I = dyn_cast(User))

664 if (!isa(I) && !IsUnreachableBlock(I->getParent()))

667 DVI->eraseFromParent();

668 }

669 };

670 for_each(Worklist, RemoveOne);

671 for_each(DbgVariableRecords, RemoveOne);

672}

673

675

676

677

678

679

681 auto *OldEntry = &NewF->getEntryBlock();

682 Entry->setName("entry" + Suffix);

683 Entry->moveBefore(OldEntry);

684 Entry->getTerminator()->eraseFromParent();

685

686

687

688

689 assert(Entry->hasOneUse());

690 auto BranchToEntry = cast(Entry->user_back());

691 assert(BranchToEntry->isUnconditional());

692 Builder.SetInsertPoint(BranchToEntry);

693 Builder.CreateUnreachable();

694 BranchToEntry->eraseFromParent();

695

696

697 Builder.SetInsertPoint(Entry);

700

701

702 auto *SwitchBB =

704 Builder.CreateBr(SwitchBB);

705 break;

706 }

710

711

712

714 isa(ActiveSuspend)) ||

717 isa(ActiveSuspend)));

718 auto *MappedCS = cast(VMap[ActiveSuspend]);

719 auto Branch = cast(MappedCS->getNextNode());

720 assert(Branch->isUnconditional());

721 Builder.CreateBr(Branch->getSuccessor(0));

722 break;

723 }

724 }

725

726

727

728 Function *F = OldEntry->getParent();

731 auto *Alloca = dyn_cast(&I);

732 if (!Alloca || I.use_empty())

733 continue;

735 !isa(Alloca->getArraySize()))

736 continue;

737 I.moveBefore(*Entry, Entry->getFirstInsertionPt());

738 }

739}

740

741

743

744

746

748 return &*NewF->arg_begin();

749

750

751

752

753

755 auto *ActiveAsyncSuspend = cast(ActiveSuspend);

756 auto ContextIdx = ActiveAsyncSuspend->getStorageArgumentIndex() & 0xff;

757 auto *CalleeContext = NewF->getArg(ContextIdx);

758 auto *ProjectionFunc =

759 ActiveAsyncSuspend->getAsyncContextProjectionFunction();

760 auto DbgLoc =

761 cast(VMap[ActiveSuspend])->getDebugLoc();

762

763 auto *CallerContext = Builder.CreateCall(ProjectionFunc->getFunctionType(),

764 ProjectionFunc, CalleeContext);

765 CallerContext->setCallingConv(ProjectionFunc->getCallingConv());

766 CallerContext->setDebugLoc(DbgLoc);

767

768 auto &Context = Builder.getContext();

769 auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(

772

775 assert(InlineRes.isSuccess());

776 (void)InlineRes;

777 return FramePtrAddr;

778 }

779

782 Argument *NewStorage = &*NewF->arg_begin();

784

785

787 return NewStorage;

788

789

790 return Builder.CreateLoad(FramePtrTy, NewStorage);

791 }

792 }

794}

795

796

797

798

799

800

801

804 if (!ActiveSuspend)

805 return;

806

808

809

810 if (auto *Branch = dyn_cast_or_null(Successor);

811 Branch && Branch->isUnconditional())

812 Successor = Branch->getSuccessor(0)->getFirstNonPHIOrDbg();

813

814

815

818 if ( || DL.getLine() == 0)

819 continue;

820

821 if (SPToUpdate.getFile() == DL->getFile()) {

822 SPToUpdate.setScopeLine(DL.getLine());

823 return;

824 }

825

826 break;

827 }

828

829

831 if (SPToUpdate.getFile() == DL->getFile())

832 SPToUpdate.setScopeLine(DL->getLine());

833}

834

837 Align Alignment, bool NoAlias) {

841

842 if (NoAlias)

844

847 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);

848}

849

851 unsigned ParamIndex) {

853 ParamAttrs.addAttribute(Attribute::SwiftAsync);

854 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);

855}

856

858 unsigned ParamIndex) {

860 ParamAttrs.addAttribute(Attribute::SwiftSelf);

861 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);

862}

863

864

865

868

869

870

871

872

873

875 for (Argument &A : OrigF.args()) {

877 VMap[&A] = DummyArgs.back();

878 }

879

881

882

883

884 auto savedVisibility = NewF->getVisibility();

885 auto savedUnnamedAddr = NewF->getUnnamedAddr();

886 auto savedDLLStorageClass = NewF->getDLLStorageClass();

887

888

889

890

891 auto savedLinkage = NewF->getLinkage();

893

896

897 auto &Context = NewF->getContext();

898

900 assert(SP != OrigF.getSubprogram() && SP->isDistinct());

902

903

904

905

906

907

908

909

910

911 if (SP->getUnit() &&

912 SP->getUnit()->getSourceLanguage() == dwarf::DW_LANG_Swift) {

913 SP->replaceLinkageName(MDString::get(Context, NewF->getName()));

914 if (auto *Decl = SP->getDeclaration()) {

916 Decl->getContext(), Decl->getScope(), Decl->getName(),

917 NewF->getName(), Decl->getFile(), Decl->getLine(), Decl->getType(),

918 Decl->getScopeLine(), Decl->getContainingType(),

919 Decl->getVirtualIndex(), Decl->getThisAdjustment(),

920 Decl->getFlags(), Decl->getSPFlags(), Decl->getUnit(),

921 Decl->getTemplateParams(), nullptr, Decl->getRetainedNodes(),

922 Decl->getThrownTypes(), Decl->getAnnotations(),

923 Decl->getTargetFuncName());

924 SP->replaceDeclaration(NewDecl);

925 }

926 }

927 }

928

929 NewF->setLinkage(savedLinkage);

930 NewF->setVisibility(savedVisibility);

931 NewF->setUnnamedAddr(savedUnnamedAddr);

932 NewF->setDLLStorageClass(savedDLLStorageClass);

933

934

935

937 NewF->hasMetadata(LLVMContext::MD_func_sanitize))

938 NewF->eraseMetadata(LLVMContext::MD_func_sanitize);

939

940

941 auto OrigAttrs = NewF->getAttributes();

943

946

947

948 NewAttrs = NewAttrs.addFnAttributes(

949 Context, AttrBuilder(Context, OrigAttrs.getFnAttrs()));

950

953 break;

955 auto *ActiveAsyncSuspend = cast(ActiveSuspend);

957 Attribute::SwiftAsync)) {

958 uint32_t ArgAttributeIndices =

959 ActiveAsyncSuspend->getStorageArgumentIndex();

960 auto ContextArgIndex = ArgAttributeIndices & 0xff;

962

963

964

965 auto SwiftSelfIndex = ArgAttributeIndices >> 8;

966 if (SwiftSelfIndex)

968 }

969

970

971 auto FnAttrs = OrigF.getAttributes().getFnAttrs();

972 NewAttrs = NewAttrs.addFnAttributes(Context, AttrBuilder(Context, FnAttrs));

973 break;

974 }

977

978

980

981

985 true);

986

987 break;

988 }

989

991

992

993

994

997

1000 break;

1001

1002

1003

1004

1006 break;

1007

1008

1009

1010

1012 break;

1013 }

1014

1015 NewF->setAttributes(NewAttrs);

1017

1018

1019 replaceEntryBlock();

1020

1021

1023 ResumeCall = cast(VMap[ResumeCall]);

1025

1026

1028 }

1029

1030

1031

1035 Builder.CreateRetVoid();

1037 }

1038

1039 Builder.SetInsertPoint(&NewF->getEntryBlock().front());

1040 NewFramePtr = deriveNewFramePointer();

1041

1042

1044 NewFramePtr->takeName(OldFramePtr);

1046

1047

1048 auto *NewVFrame = Builder.CreateBitCast(

1051 if (OldVFrame != NewVFrame)

1053

1054

1055

1056 for (Instruction *DummyArg : DummyArgs) {

1057 DummyArg->replaceAllUsesWith(PoisonValue::get(DummyArg->getType()));

1058 DummyArg->deleteValue();

1059 }

1060

1063

1064

1065

1067 handleFinalSuspend();

1068 break;

1072

1073

1074 assert(ActiveSuspend != nullptr &&

1075 "no active suspend when lowering a continuation-style coroutine");

1076 replaceRetconOrAsyncSuspendUses();

1077 break;

1078 }

1079

1080

1081 replaceCoroSuspends();

1082

1083

1085

1086

1087 replaceCoroEnds();

1088

1089

1091}

1092

1094

1096 ActiveSuspend);

1097

1098

1100

1101

1102

1105}

1106

1109

1110 auto *FuncPtrStruct = cast(

1112 auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0);

1113 auto *OrigContextSize = FuncPtrStruct->getOperand(1);

1114 auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(),

1117 FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize);

1118

1120}

1121

1123

1124 auto *SizeIntrin = Shape.CoroSizes.back();

1125 Module *M = SizeIntrin->getModule();

1127 return DL.getTypeAllocSize(Shape.FrameTy);

1128}

1129

1133

1138 }

1139

1141 return;

1142

1143

1144 auto *SizeIntrin = Shape.CoroSizes.back();

1145 auto *SizeConstant =

1147

1151 }

1152}

1153

1156

1157#ifndef NDEBUG

1158

1159

1160

1163#endif

1164}

1165

1166

1167

1169 auto *CoroBegin = Shape.CoroBegin;

1170 switch (Shape.ABI) {

1175 if (AllocInst) {

1179 AllocInst->replaceAllUsesWith(Builder.getFalse());

1180 AllocInst->eraseFromParent();

1181 CoroBegin->replaceAllUsesWith(Frame);

1182 } else {

1183 CoroBegin->replaceAllUsesWith(CoroBegin->getMem());

1184 }

1185

1186 break;

1187 }

1191 CoroBegin->replaceAllUsesWith(PoisonValue::get(CoroBegin->getType()));

1192 break;

1193 }

1194

1195 CoroBegin->eraseFromParent();

1197}

1198

1199

1200

1201

1204

1205 if (isa(I))

1206 continue;

1207

1208 if (isa(I))

1209 return true;

1210 }

1211 return false;

1212}

1213

1217

1218 Set.insert(SaveBB);

1220

1221

1222

1223

1224 while (!Worklist.empty()) {

1226 Set.insert(BB);

1228 if (!Set.contains(Pred))

1230 }

1231

1232

1233 Set.erase(SaveBB);

1234 Set.erase(ResDesBB);

1235

1236 for (auto *BB : Set)

1238 return true;

1239

1240 return false;

1241}

1242

1244 auto *SaveBB = Save->getParent();

1245 auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();

1246

1247 if (SaveBB == ResumeOrDestroyBB)

1249

1250

1252 return true;

1253

1254

1256 ResumeOrDestroy))

1257 return true;

1258

1259

1261 return true;

1262

1263 return false;

1264}

1265

1266

1267

1271 if (!Prev) {

1272 auto *Pred = Suspend->getParent()->getSinglePredecessor();

1273 if (!Pred)

1274 return false;

1275 Prev = Pred->getTerminator();

1276 }

1277

1278 CallBase *CB = dyn_cast(Prev);

1279 if (!CB)

1280 return false;

1281

1283

1284

1285 auto *SubFn = dyn_cast(Callee);

1286 if (!SubFn)

1287 return false;

1288

1289

1290 if (SubFn->getFrame() != CoroBegin)

1291 return false;

1292

1293

1294

1295

1298 return false;

1299

1300

1301

1304 Save->eraseFromParent();

1305

1306

1307 if (auto *Invoke = dyn_cast(CB)) {

1309 }

1310

1311

1314

1315

1316 if (CalledValue != SubFn && CalledValue->user_empty())

1317 if (auto *I = dyn_cast(CalledValue))

1318 I->eraseFromParent();

1319

1320

1321 if (SubFn->user_empty())

1322 SubFn->eraseFromParent();

1323

1324 return true;

1325}

1326

1327

1329

1331 return;

1332

1334 size_t I = 0, N = S.size();

1335 if (N == 0)

1336 return;

1337

1338 size_t ChangedFinalIndex = std::numeric_limits<size_t>::max();

1339 while (true) {

1340 auto SI = cast(S[I]);

1341

1342

1344 if (--N == I)

1345 break;

1346

1348

1349 if (cast(S[I])->isFinal()) {

1351 ChangedFinalIndex = I;

1352 }

1353

1354 continue;

1355 }

1356 if (++I == N)

1357 break;

1358 }

1359 S.resize(N);

1360

1361

1362

1363 if (ChangedFinalIndex < N) {

1364 assert(cast(S[ChangedFinalIndex])->isFinal());

1365 std::swap(S[ChangedFinalIndex], S.back());

1366 }

1367}

1368

1369namespace {

1370

1371struct SwitchCoroutineSplitter {

1375 assert(Shape.ABI == coro::ABI::Switch);

1376

1377

1378

1379

1380 createResumeEntryBlock(F, Shape);

1382 F, ".resume", Shape, coro::CloneKind::SwitchResume, TTI);

1384 F, ".destroy", Shape, coro::CloneKind::SwitchUnwind, TTI);

1386 F, ".cleanup", Shape, coro::CloneKind::SwitchCleanup, TTI);

1387

1391

1392

1393 updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);

1394

1399

1400

1401

1402

1403 setCoroInfo(F, Shape, Clones);

1404 }

1405

1406

1407

1408

1409

1410

1411

1412

1415 assert(Shape.ABI == coro::ABI::Switch);

1416 auto *OrigFnTy = F.getFunctionType();

1417 auto OldParams = OrigFnTy->params();

1418

1420 NewParams.reserve(OldParams.size() + 1);

1421 NewParams.append(OldParams.begin(), OldParams.end());

1423

1424 auto *NewFnTy = FunctionType::get(OrigFnTy->getReturnType(), NewParams,

1425 OrigFnTy->isVarArg());

1427 Function::Create(NewFnTy, F.getLinkage(), F.getName() + ".noalloc");

1428

1430 unsigned int Idx = 0;

1431 for (const auto &I : F.args()) {

1433 }

1434

1435

1436 auto FrameIdx = NoAllocF->arg_size() - 1;

1439 CloneFunctionChangeType::LocalChangesOnly, Returns);

1440

1442 auto *NewCoroBegin =

1443 cast_if_present(VMap[Shape.CoroBegin]);

1444 auto *NewCoroId = cast(NewCoroBegin->getId());

1447 NewCoroBegin->replaceAllUsesWith(NoAllocF->getArg(FrameIdx));

1448 NewCoroBegin->eraseFromParent();

1449 }

1450

1452 M->getFunctionList().insert(M->end(), NoAllocF);

1453

1456

1457

1460 false);

1461

1463

1465

1466

1467 setCoroInfo(F, Shape, Clones);

1468

1469

1470

1472 return NoAllocF;

1473 }

1474

1475private:

1476

1477

1480

1481

1482

1483

1484

1485

1486

1487

1488

1489

1492

1495 auto *FrameTy = Shape.FrameTy;

1496 auto *GepIndex = Builder.CreateStructGEP(

1498 auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");

1500 Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());

1502

1503 size_t SuspendIndex = 0;

1505 auto *S = cast(AnyS);

1507

1508

1509

1510

1511 auto *Save = S->getCoroSave();

1512 Builder.SetInsertPoint(Save);

1513 if (S->isFinal()) {

1514

1515

1517 } else {

1518 auto *GepIndex = Builder.CreateStructGEP(

1520 Builder.CreateStore(IndexVal, GepIndex);

1521 }

1522

1524 Save->eraseFromParent();

1525

1526

1527

1528

1529

1530

1531

1532

1533

1534

1535

1536

1537

1538

1539

1540

1541

1542

1543

1544

1545

1546

1547

1548

1549 auto *SuspendBB = S->getParent();

1550 auto *ResumeBB =

1551 SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex));

1552 auto *LandingBB = ResumeBB->splitBasicBlock(

1553 S->getNextNode(), ResumeBB->getName() + Twine(".landing"));

1554 Switch->addCase(IndexVal, ResumeBB);

1555

1556 cast(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);

1558 PN->insertBefore(LandingBB->begin());

1559 S->replaceAllUsesWith(PN);

1560 PN->addIncoming(Builder.getInt8(-1), SuspendBB);

1561 PN->addIncoming(S, ResumeBB);

1562

1563 ++SuspendIndex;

1564 }

1565

1566 Builder.SetInsertPoint(UnreachBB);

1567 Builder.CreateUnreachable();

1568

1570 }

1571

1572

1576

1577 auto *ResumeAddr = Builder.CreateStructGEP(

1579 "resume.addr");

1580 Builder.CreateStore(ResumeFn, ResumeAddr);

1581

1582 Value *DestroyOrCleanupFn = DestroyFn;

1583

1586

1587

1588 DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);

1589 }

1590

1591 auto *DestroyAddr = Builder.CreateStructGEP(

1593 "destroy.addr");

1594 Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);

1595 }

1596

1597

1598

1599

1600

1601

1602

1603

1604

1605

1606

1607

1608

1609

1612

1613

1618 auto *ArrTy = ArrayType::get(Part->getType(), Args.size());

1619

1621 auto *GV = new GlobalVariable(*M, ConstVal->getType(), true,

1622 GlobalVariable::PrivateLinkage, ConstVal,

1623 F.getName() + Twine(".resumers"));

1624

1625

1629 }

1630};

1631

1632}

1633

1637 auto &Context = Suspend->getParent()->getParent()->getContext();

1639

1643 ResumeIntrinsic->eraseFromParent();

1646}

1647

1648

1652 size_t ArgIdx = 0;

1653 for (auto *paramTy : FnTy->params()) {

1655 if (paramTy != FnArgs[ArgIdx]->getType())

1658 else

1659 CallArgs.push_back(FnArgs[ArgIdx]);

1660 ++ArgIdx;

1661 }

1662}

1663

1669

1670

1673

1674 auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs);

1675

1678 }

1679 TailCall->setDebugLoc(Loc);

1680 TailCall->setCallingConv(MustTailCallFn->getCallingConv());

1681 return TailCall;

1682}

1683

1689

1690

1691 F.removeFnAttr(Attribute::NoReturn);

1692 F.removeRetAttr(Attribute::NoAlias);

1693 F.removeRetAttr(Attribute::NonNull);

1694

1695 auto &Context = F.getContext();

1697

1700

1701 auto *FramePtr = Id->getStorage();

1705 "async.ctx.frameptr");

1706

1707

1708 {

1709

1713 }

1714

1715

1716 auto NextF = std::next(F.getIterator());

1717

1718

1721 auto *Suspend = cast(CS);

1722

1723

1724 auto ResumeNameSuffix = ".resume.";

1725 auto ProjectionFunctionName =

1726 Suspend->getAsyncContextProjectionFunction()->getName();

1727 bool UseSwiftMangling = false;

1728 if (ProjectionFunctionName == "__swift_async_resume_project_context") {

1729 ResumeNameSuffix = "TQ";

1730 UseSwiftMangling = true;

1731 } else if (ProjectionFunctionName == "__swift_async_resume_get_context") {

1732 ResumeNameSuffix = "TY";

1733 UseSwiftMangling = true;

1734 }

1737 UseSwiftMangling ? ResumeNameSuffix + Twine(Idx) + "_"

1738 : ResumeNameSuffix + Twine(Idx),

1739 NextF, Suspend);

1741

1742

1743

1744 auto *SuspendBB = Suspend->getParent();

1745 auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);

1746 auto *Branch = cast(SuspendBB->getTerminator());

1747

1748

1749 auto *ReturnBB =

1751 Branch->setSuccessor(0, ReturnBB);

1752

1754

1755

1756 auto *Fn = Suspend->getMustTailCallFunction();

1761 FnArgs, Builder);

1765

1766

1768 }

1769

1772 auto *Suspend = CS;

1773 auto *Clone = Clones[Idx];

1774

1776 Suspend, TTI);

1777 }

1778}

1779

1785

1786

1787

1788 F.removeFnAttr(Attribute::NoReturn);

1789 F.removeRetAttr(Attribute::NoAlias);

1790 F.removeRetAttr(Attribute::NonNull);

1791

1792

1794 Value *RawFramePtr;

1796 RawFramePtr = Id->getStorage();

1797 } else {

1799

1800

1803

1804

1805

1806

1808 RawFramePtr =

1810

1811

1812 Builder.CreateStore(RawFramePtr, Id->getStorage());

1813 }

1814

1815

1816 {

1817

1821 }

1822

1823

1825 PHINode *ContinuationPhi = nullptr;

1827

1828

1829 auto NextF = std::next(F.getIterator());

1830

1831

1834 auto Suspend = cast(CS);

1835

1836

1840

1841

1842

1843 auto SuspendBB = Suspend->getParent();

1844 auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);

1845 auto Branch = cast(SuspendBB->getTerminator());

1846

1847

1848 if (!ReturnBB) {

1849

1850 ReturnBB =

1853

1855

1856

1857 ContinuationPhi =

1859

1860

1862

1863

1867

1868

1869 auto RetTy = F.getReturnType();

1870

1871

1872

1873

1874 auto CastedContinuationTy =

1875 (ReturnPHIs.empty() ? RetTy : RetTy->getStructElementType(0));

1876 auto *CastedContinuation =

1877 Builder.CreateBitCast(ContinuationPhi, CastedContinuationTy);

1878

1879 Value *RetV = CastedContinuation;

1880 if (!ReturnPHIs.empty()) {

1881 auto ValueIdx = 0;

1883 RetV = Builder.CreateInsertValue(RetV, CastedContinuation, ValueIdx++);

1884

1885 for (auto Phi : ReturnPHIs)

1887 }

1888

1890 }

1891

1892

1893 Branch->setSuccessor(0, ReturnBB);

1894 assert(ContinuationPhi);

1896 for (auto [Phi, VUse] :

1898 Phi->addIncoming(VUse, SuspendBB);

1899 }

1900

1903 auto Suspend = CS;

1904 auto Clone = Clones[Idx];

1905

1907 Suspend, TTI);

1908 }

1909}

1910

1911namespace {

1914

1915public:

1916 PrettyStackTraceFunction(Function &F) : F(F) {}

1918 OS << "While splitting coroutine ";

1919 F.printAsOperand(OS, false, F.getParent());

1920 OS << "\n";

1921 }

1922};

1923}

1924

1925

1930 }

1931 } else {

1933 auto &Context = End->getContext();

1935 End->eraseFromParent();

1936 }

1937 }

1938}

1939

1941 for (auto *U : F.users()) {

1942 if (auto *CB = dyn_cast(U)) {

1943 auto *Caller = CB->getFunction();

1944 if (Caller && Caller->isPresplitCoroutine() &&

1945 CB->hasFnAttr(llvm::Attribute::CoroElideSafe))

1946 return true;

1947 }

1948 }

1949 return false;

1950}

1951

1955 SwitchCoroutineSplitter::split(F, Shape, Clones, TTI);

1956}

1957

1960 bool OptimizeFrame) {

1961 PrettyStackTraceFunction prettyStackTrace(F);

1962

1963 auto &Shape = ABI.Shape;

1965

1967

1969

1971 ABI.buildCoroutineFrame(OptimizeFrame);

1973

1974 bool isNoSuspendCoroutine = Shape.CoroSuspends.empty();

1975

1976 bool shouldCreateNoAllocVariant =

1979

1980

1981

1982 if (isNoSuspendCoroutine) {

1984 } else {

1985 ABI.splitCoroutine(F, Shape, Clones, TTI);

1986 }

1987

1988

1989

1991

1992

1993

1994

1997 for (auto *DDI : DbgInsts)

2001

2003

2004 if (shouldCreateNoAllocVariant)

2005 SwitchCoroutineSplitter::createNoAllocVariant(F, Shape, Clones);

2006}

2007

2013

2014 auto *CurrentSCC = &C;

2015 if (!Clones.empty()) {

2016 switch (Shape.ABI) {

2018

2019

2020 for (Function *Clone : Clones)

2022 break;

2026

2027

2028 if (!Clones.empty())

2030 break;

2031 }

2032

2033

2035 UR, FAM);

2036 }

2037

2038

2039

2042 AM, UR, FAM);

2043 return *CurrentSCC;

2044}

2045

2046

2049 auto CastFn = Prepare->getArgOperand(0);

2051

2052

2053

2054

2055

2056

2057

2059

2060 auto *Cast = dyn_cast(U.getUser());

2061 if (!Cast || Cast->getType() != Fn->getType())

2062 continue;

2063

2064

2065 Cast->replaceAllUsesWith(Fn);

2066 Cast->eraseFromParent();

2067 }

2068

2069

2070

2073

2074

2075 while (auto *Cast = dyn_cast(CastFn)) {

2076 if (!Cast->use_empty())

2077 break;

2078 CastFn = Cast->getOperand(0);

2079 Cast->eraseFromParent();

2080 }

2081}

2082

2085 bool Changed = false;

2087

2088 auto *Prepare = cast(P.getUser());

2090 Changed = true;

2091 }

2092

2093 return Changed;

2094}

2095

2099 auto *PrepareFn = M.getFunction(Name);

2100 if (PrepareFn && !PrepareFn->use_empty())

2102}

2103

2104static std::unique_ptrcoro::BaseABI

2106 std::function<bool(Instruction &)> IsMatCallback,

2110 if (CustomABI >= GenCustomABIs.size())

2111 llvm_unreachable("Custom ABI not found amoung those specified");

2112 return GenCustomABIs[CustomABI](F, S);

2113 }

2114

2115 switch (S.ABI) {

2117 return std::make_uniquecoro::SwitchABI(F, S, IsMatCallback);

2119 return std::make_uniquecoro::AsyncABI(F, S, IsMatCallback);

2121 return std::make_uniquecoro::AnyRetconABI(F, S, IsMatCallback);

2123 return std::make_uniquecoro::AnyRetconABI(F, S, IsMatCallback);

2124 }

2126}

2127

2129 : CreateAndInitABI([](Function &F, coro::Shape &S) {

2130 std::unique_ptrcoro::BaseABI ABI =

2132 ABI->init();

2133 return ABI;

2134 }),

2135 OptimizeFrame(OptimizeFrame) {}

2136

2139 : CreateAndInitABI([=](Function &F, coro::Shape &S) {

2140 std::unique_ptrcoro::BaseABI ABI =

2142 ABI->init();

2143 return ABI;

2144 }),

2145 OptimizeFrame(OptimizeFrame) {}

2146

2147

2148

2150 bool OptimizeFrame)

2151 : CreateAndInitABI([=](Function &F, coro::Shape &S) {

2152 std::unique_ptrcoro::BaseABI ABI =

2154 ABI->init();

2155 return ABI;

2156 }),

2157 OptimizeFrame(OptimizeFrame) {}

2158

2159

2160

2162 std::function<bool(Instruction &)> IsMatCallback,

2164 : CreateAndInitABI([=](Function &F, coro::Shape &S) {

2165 std::unique_ptrcoro::BaseABI ABI =

2166 CreateNewABI(F, S, IsMatCallback, GenCustomABIs);

2167 ABI->init();

2168 return ABI;

2169 }),

2170 OptimizeFrame(OptimizeFrame) {}

2171

2175

2176

2177

2178 Module &M = *C.begin()->getFunction().getParent();

2179 auto &FAM =

2181

2182

2186

2187

2190 if (N.getFunction().isPresplitCoroutine())

2192

2193 if (Coroutines.empty() && PrepareFns.empty())

2195

2196 auto *CurrentSCC = &C;

2197

2200 LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()

2201 << "\n");

2202

2203

2204

2205

2207

2210 continue;

2211

2212 F.setSplittedCoroutine();

2213

2214 std::unique_ptrcoro::BaseABI ABI = CreateAndInitABI(F, Shape);

2215

2220 *N, Shape, Clones, *CurrentSCC, CG, AM, UR, FAM);

2221

2223 ORE.emit([&]() {

2225 << "Split '" << ore::NV("function", F.getName())

2226 << "' (frame_size=" << ore::NV("frame_size", Shape.FrameSize)

2228 });

2229

2231

2232 UR.CWorklist.insert(CurrentSCC);

2233 for (Function *Clone : Clones)

2235 }

2236 }

2237

2238 for (auto *PrepareFn : PrepareFns) {

2240 }

2241

2243}

amdgpu aa AMDGPU Address space based Alias Analysis Wrapper

AMDGPU Lower Kernel Arguments

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)

Expand Atomic instructions

This file contains the simple types necessary to represent the attributes associated with functions a...

BlockVerifier::State From

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

This file provides interfaces used to manipulate a call graph, regardless if it is a "old style" Call...

This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...

This file contains the declarations for the subclasses of Constant, which represent the different fla...

static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)

static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy)

static std::pair< SmallVector< DbgVariableIntrinsic *, 8 >, SmallVector< DbgVariableRecord * > > collectDbgVariableIntrinsics(Function &F)

Returns all DbgVariableIntrinsic in F.

static LazyCallGraph::SCC & updateCallGraphAfterCoroutineSplit(LazyCallGraph::Node &N, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, LazyCallGraph::SCC &C, LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)

static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)

static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)

static void maybeFreeRetconStorage(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG)

static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB)

static Function * createCloneDeclaration(Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)

Remove calls to llvm coro end in the original static function void removeCoroEndsFromRampFunction(const coro::Shape &Shape)

static FunctionType * getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend)

static void updateScopeLine(Instruction *ActiveSuspend, DISubprogram &SPToUpdate)

Adjust the scope line of the funclet to the first line number after the suspend point.

static void addPrepareFunction(const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)

static void simplifySuspendPoints(coro::Shape &Shape)

static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment, bool NoAlias)

static bool hasSafeElideCaller(Function &F)

static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)

static void replaceFallthroughCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)

Replace a non-unwind call to llvm.coro.end.

static void replaceFrameSizeAndAlignment(coro::Shape &Shape)

static std::unique_ptr< coro::BaseABI > CreateNewABI(Function &F, coro::Shape &S, std::function< bool(Instruction &)> IsMatCallback, const SmallVector< CoroSplitPass::BaseABITy > GenCustomABIs)

static bool replaceCoroEndAsync(AnyCoroEndInst *End)

Replace an llvm.coro.end.async.

static void doSplitCoroutine(Function &F, SmallVectorImpl< Function * > &Clones, coro::BaseABI &ABI, TargetTransformInfo &TTI, bool OptimizeFrame)

Replace a call to llvm coro prepare static retcon void replacePrepare(CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)

static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)

Replace an unwind call to llvm.coro.end.

static bool simplifySuspendPoint(CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)

static bool hasCallsInBlockBetween(Instruction *From, Instruction *To)

static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr)

static void updateAsyncFuncPointerContextSize(coro::Shape &Shape)

static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)

static void lowerAwaitSuspend(IRBuilder<> &Builder, CoroAwaitSuspendInst *CB, coro::Shape &Shape)

static void lowerAwaitSuspends(Function &F, coro::Shape &Shape)

static void handleNoSuspendCoroutine(coro::Shape &Shape)

static void postSplitCleanup(Function &F)

static TypeSize getFrameSizeForShape(coro::Shape &Shape)

Coerce the arguments in p FnArgs according to p FnTy in p static CallArgs void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > &CallArgs)

static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend, Value *Continuation)

Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx

This file defines the DenseMap class.

This file contains constants used for implementing Dwarf debug support.

This file provides various utilities for inspecting and working with the control flow graph in LLVM I...

Module.h This file contains the declarations for the Module class.

Implements a lazy call graph analysis and related passes for the new pass manager.

FunctionAnalysisManager FAM

This file provides a priority worklist.

const SmallVectorImpl< MachineOperand > & Cond

Remove Loads Into Fake Uses

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This file defines the SmallPtrSet class.

This file defines the SmallVector class.

static SymbolRef::Type getType(const Symbol *Sym)

This pass exposes codegen information to IR-level passes.

static const unsigned FramePtr

void setSwiftError(bool V)

Specify whether this alloca is used to represent a swifterror.

void setAlignment(Align Align)

A container for analyses that lazily runs them and caches their results.

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

CoroAllocInst * getCoroAlloc()

Align getStorageAlignment() const

uint64_t getStorageSize() const

This class represents an incoming formal argument to a Function.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

ArrayRef< T > drop_front(size_t N=1) const

Drop the first N elements of the array.

size_t size() const

size - Get the array size.

AttrBuilder & addAlignmentAttr(MaybeAlign Align)

This turns an alignment into the form used internally in Attribute.

AttrBuilder & addAttribute(Attribute::AttrKind Val)

Add an attribute to the builder.

AttrBuilder & addDereferenceableAttr(uint64_t Bytes)

This turns the number of dereferenceable bytes into the form used internally in Attribute.

AttributeList removeParamAttributes(LLVMContext &C, unsigned ArgNo, const AttributeMask &AttrsToRemove) const

Remove the specified attribute at the specified arg index from this attribute list.

LLVM Basic Block Representation.

static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)

Creates a new BasicBlock.

BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)

Split the basic block into two basic blocks at the specified instruction.

const Function * getParent() const

Return the enclosing method, or null if none.

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

void setCallingConv(CallingConv::ID CC)

Function * getCalledFunction() const

Returns the function called, or null if this is an indirect function invocation or the function signa...

Value * getCalledOperand() const

void setAttributes(AttributeList A)

Set the attributes for this call.

Value * getArgOperand(unsigned i) const

AttributeList getAttributes() const

Return the attributes for this call.

The basic data container for the call graph of a Module of IR.

This class represents a function call, abstracting a target machine's calling convention.

static Constant * get(ArrayType *T, ArrayRef< Constant * > V)

static Constant * getPointerCast(Constant *C, Type *Ty)

Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.

This is the shared class of boolean and integer constants.

static ConstantInt * getTrue(LLVMContext &Context)

static ConstantInt * getFalse(LLVMContext &Context)

static ConstantPointerNull * get(PointerType *T)

Static factory methods - Return objects of the specified value.

static Constant * get(StructType *T, ArrayRef< Constant * > V)

static ConstantTokenNone * get(LLVMContext &Context)

Return the ConstantTokenNone.

This represents the llvm.coro.align instruction.

This represents the llvm.coro.alloc instruction.

This represents the llvm.coro.await.suspend.{void,bool,handle} instructions.

Value * getAwaiter() const

Function * getWrapperFunction() const

This class represents the llvm.coro.begin or llvm.coro.begin.custom.abi instructions.

AnyCoroIdInst * getId() const

bool hasCustomABI() const

This represents the llvm.coro.id instruction.

void setInfo(Constant *C)

This represents the llvm.coro.size instruction.

This represents the llvm.coro.suspend.async instruction.

CoroAsyncResumeInst * getResumeFunction() const

This represents the llvm.coro.suspend instruction.

CoroSaveInst * getCoroSave() const

DISubprogram * getSubprogram() const

Get the subprogram for this scope.

This class represents an Operation in the Expression.

A parsed version of the target data layout string in and methods for querying it.

This is the common base class for debug info intrinsics for variables.

Record of a variable value-assignment, aka a non instruction representation of the dbg....

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

bool isReachableFromEntry(const Use &U) const

Provide an overload for a Use.

This class represents a freeze function that returns random concrete value if an operand is either a ...

A proxy from a FunctionAnalysisManager to an SCC.

Class to represent function types.

Type * getReturnType() const

static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)

FunctionType * getFunctionType() const

Returns the FunctionType for me.

Intrinsic::ID getIntrinsicID() const LLVM_READONLY

getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...

CallingConv::ID getCallingConv() const

getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...

AttributeList getAttributes() const

Return the attribute list for this Function.

void setAttributes(AttributeList Attrs)

Set the attribute list for this Function.

LLVMContext & getContext() const

getContext - Return a reference to the LLVMContext associated with this function.

bool isCoroOnlyDestroyWhenComplete() const

Argument * getArg(unsigned i) const

void setLinkage(LinkageTypes LT)

Module * getParent()

Get the module that this global value is contained inside of...

PointerType * getType() const

Global values are always pointers.

@ InternalLinkage

Rename collisions when linking (static functions).

@ ExternalLinkage

Externally visible function.

const Constant * getInitializer() const

getInitializer - Return the initializer for this global variable.

void setInitializer(Constant *InitVal)

setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...

AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")

Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")

InvokeInst * CreateInvoke(FunctionType *Ty, Value *Callee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef< Value * > Args, ArrayRef< OperandBundleDef > OpBundles, const Twine &Name="")

Create an invoke instruction.

BasicBlock::iterator GetInsertPoint() const

Value * CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx, const Twine &Name="")

Value * CreateConstInBoundsGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")

CleanupReturnInst * CreateCleanupRet(CleanupPadInst *CleanupPad, BasicBlock *UnwindBB=nullptr)

ReturnInst * CreateRet(Value *V)

Create a 'ret ' instruction.

ConstantInt * getInt64(uint64_t C)

Get a constant 64-bit value.

Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")

PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")

Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")

BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)

Create a conditional 'br Cond, TrueDest, FalseDest' instruction.

LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)

Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...

LLVMContext & getContext() const

ReturnInst * CreateRetVoid()

Create a 'ret void' instruction.

StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)

ConstantInt * getFalse()

Get the constant value for i1 false.

CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)

BranchInst * CreateBr(BasicBlock *Dest)

Create an unconditional 'br label X' instruction.

Value * CreateIsNull(Value *Arg, const Twine &Name="")

Return a boolean value testing if Arg == 0.

void SetInsertPoint(BasicBlock *TheBB)

This specifies that created instructions should be appended to the end of the specified block.

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

This class captures the data input to the InlineFunction call, and records the auxiliary results prod...

const DebugLoc & getDebugLoc() const

Return the debug location for this node as a DebugLoc.

InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

const Instruction * getNextNonDebugInstruction(bool SkipPseudoOp=false) const

Return a pointer to the next non-debug instruction in the same basic block as 'this',...

This is an important class for using LLVM in a threaded context.

A node in the call graph.

An SCC of the call graph.

A lazily constructed view of the call graph of a module.

void addSplitFunction(Function &OriginalFunction, Function &NewFunction)

Add a new function split/outlined from an existing function.

void addSplitRefRecursiveFunctions(Function &OriginalFunction, ArrayRef< Function * > NewFunctions)

Add new ref-recursive functions split/outlined from an existing function.

Node & get(Function &F)

Get a graph node for a given function, scanning it to populate the graph data as necessary.

SCC * lookupSCC(Node &N) const

Lookup a function's SCC in the graph.

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

static MDString * get(LLVMContext &Context, StringRef Str)

A Module instance is used to store all the information related to an LLVM module.

FunctionListType::iterator iterator

The Function iterators.

void addIncoming(Value *V, BasicBlock *BB)

Add an incoming value to the end of the PHI list.

static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...

static PointerType * getUnqual(Type *ElementType)

This constructs a pointer to an object of the specified type in the default address space (address sp...

static PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

PrettyStackTraceEntry - This class is used to represent a frame of the "pretty" stack trace that is d...

Return a value (possibly void), from a function.

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void reserve(size_type N)

void append(ItTy in_start, ItTy in_end)

Add the specified range to the end of the SmallVector.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

Type * getTypeAtIndex(const Value *V) const

Given an index value into the type, return the type of the element.

Analysis pass providing the TargetTransformInfo.

This pass provides access to the codegen interfaces that are needed for IR-level transformations.

bool supportsTailCallFor(const CallBase *CB) const

If target supports tail call on CB.

Value handle that tracks a Value across RAUW.

ValueTy * getValPtr() const

Triple - Helper class for working with autoconf configuration names.

bool isArch64Bit() const

Test whether the architecture is 64-bit.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

static Type * getVoidTy(LLVMContext &C)

LLVMContext & getContext() const

Return the LLVMContext in which this type was uniqued.

static IntegerType * getInt8Ty(LLVMContext &C)

A Use represents the edge between a Value definition and its users.

void setOperand(unsigned i, Value *Val)

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

iterator_range< user_iterator > users()

const Value * stripPointerCasts() const

Strip off pointer casts, all-zero GEPs and address space casts.

LLVMContext & getContext() const

All values hold a context through their type.

iterator_range< use_iterator > uses()

StringRef getName() const

Return a constant reference to the value's name.

void takeName(Value *V)

Transfer the name from V to this value.

void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI) override

void replaceSwiftErrorOps()

Value * deriveNewFramePointer()

Derive the value of the new frame pointer.

void replaceCoroSuspends()

void handleFinalSuspend()

static Function * createClone(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, Function *NewF, AnyCoroSuspendInst *ActiveSuspend, TargetTransformInfo &TTI)

Create a clone for a continuation lowering.

bool isSwitchDestroyFunction()

void replaceRetconOrAsyncSuspendUses()

Replace uses of the active llvm.coro.suspend.retcon/async call with the arguments to the continuation...

virtual void create()

Clone the body of the original function into a resume function of some sort.

void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI) override

static Function * createClone(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, CloneKind FKind, TargetTransformInfo &TTI)

Create a clone for a switch lowering.

void create() override

Clone the body of the original function into a resume function of some sort.

const ParentTy * getParent() const

NodeTy * getNextNode()

Get the next node, or nullptr for the list tail.

This class implements an extremely fast bulk output stream that can only output to a stream.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

constexpr char Args[]

Key for Kernel::Metadata::mArgs.

@ Fast

Attempts to make calls as fast as possible (e.g.

@ C

The default llvm calling convention, compatible with C.

void salvageDebugInfo(SmallDenseMap< Argument *, AllocaInst *, 4 > &ArgToAllocaMap, DbgVariableIntrinsic &DVI, bool IsEntryPoint)

Attempts to rewrite the location operand of debug intrinsics in terms of the coroutine frame pointer,...

@ Async

The "async continuation" lowering, where each suspend point creates a single continuation function.

@ RetconOnce

The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...

@ Retcon

The "returned-continuation" lowering, where each suspend point creates a single continuation function...

@ Switch

The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...

void suppressCoroAllocs(CoroIdInst *CoroId)

Replaces all @llvm.coro.alloc intrinsics calls associated with a given call @llvm....

void normalizeCoroutine(Function &F, coro::Shape &Shape, TargetTransformInfo &TTI)

CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, TargetTransformInfo &TTI, ArrayRef< Value * > Arguments, IRBuilder<> &)

void replaceCoroFree(CoroIdInst *CoroId, bool Elide)

bool isTriviallyMaterializable(Instruction &I)

@ SwitchCleanup

The shared cleanup function for a switch lowering.

@ Continuation

An individual continuation function.

DiagnosticInfoOptimizationBase::Argument NV

This is an optimization pass for GlobalISel generic memory operations.

UnaryFunction for_each(R &&Range, UnaryFunction F)

Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.

detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)

zip iterator that assumes that all iteratees have the same length.

auto enumerate(FirstRange &&First, RestRanges &&...Rest)

Given two or more input ranges, returns a new range whose values are tuples (A, B,...

bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)

Check a function for errors, useful for use when debugging a pass.

LazyCallGraph::SCC & updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)

Helper to update the call graph after running a function pass.

LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)

Helper to update the call graph after running a CGSCC pass.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

void report_fatal_error(Error Err, bool gen_crash_diag=true)

Report a serious error, calling any installed error handler.

unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)

Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...

raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

DWARFExpression::Operation Op

InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)

This function inlines the called function into the basic block of the caller.

void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)

Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.

auto predecessors(const MachineBasicBlock *BB)

static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)

Filter the DbgRecord range to DbgVariableRecord types only and downcast.

bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)

Remove all blocks that can not be reached from the function's entry.

bool isPotentiallyReachable(const Instruction *From, const Instruction *To, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet=nullptr, const DominatorTree *DT=nullptr, const LoopInfo *LI=nullptr)

Determine whether instruction 'To' is reachable from 'From', without passing through any blocks in Ex...

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.

This struct is a compact representation of a valid (non-zero power of two) alignment.

uint64_t value() const

This is a hole in the type system and should not be abused.

Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...

SmallPriorityWorklist< LazyCallGraph::SCC *, 1 > & CWorklist

Worklist of the SCCs queued for processing.

PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)

CoroSplitPass(bool OptimizeFrame=false)

BaseABITy CreateAndInitABI

CallInst * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)

GlobalVariable * AsyncFuncPointer

bool IsFrameInlineInStorage

Function * ResumePrototype

SwitchInst * ResumeSwitch

BasicBlock * ResumeEntryBlock

SmallVector< CallInst *, 2 > SymmetricTransfers

SmallVector< CoroAwaitSuspendInst *, 4 > CoroAwaitSuspends

AsyncLoweringStorage AsyncLowering

FunctionType * getResumeFunctionType() const

IntegerType * getIndexType() const

AnyCoroIdRetconInst * getRetconCoroId() const

PointerType * getSwitchResumePointerType() const

CoroIdInst * getSwitchCoroId() const

SmallVector< CoroSizeInst *, 2 > CoroSizes

CallingConv::ID getResumeFunctionCC() const

SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends

Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const

Allocate memory according to the rules of the active lowering.

ConstantInt * getIndex(uint64_t Value) const

SwitchLoweringStorage SwitchLowering

CoroBeginInst * CoroBegin

BasicBlock::iterator getInsertPtAfterFramePtr() const

ArrayRef< Type * > getRetconResultTypes() const

void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const

Deallocate memory according to the rules of the active lowering.

RetconLoweringStorage RetconLowering

SmallVector< CoroAlignInst *, 2 > CoroAligns

CoroIdAsyncInst * getAsyncCoroId() const

SmallVector< AnyCoroEndInst *, 4 > CoroEnds

SmallVector< CallInst *, 2 > SwiftErrorOps

BasicBlock * AllocaSpillBlock

unsigned getSwitchIndexField() const