LLVM: lib/Transforms/Coroutines/CoroSplit.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
71#include
72#include
73#include
74#include <initializer_list>
75#include
76
77using namespace llvm;
78
79#define DEBUG_TYPE "coro-split"
80
81namespace {
82
83
84
85
86
88 TimeTraceScope FunctionScope("CollectCommonDebugInfo");
89
92 F, CloneFunctionChangeType::LocalChangesOnly, DIFinder);
93
95 DIFinder, SPClonedWithinModule);
96}
97}
98
99
100
101
102
108
110
112
113
114
117
118 if (auto Invoke = dyn_cast(CB)) {
119 auto WrapperInvoke =
121 Invoke->getUnwindDest(), {Awaiter, FramePtr});
122
123 WrapperInvoke->setCallingConv(Invoke->getCallingConv());
124 std::copy(Invoke->bundle_op_info_begin(), Invoke->bundle_op_info_end(),
125 WrapperInvoke->bundle_op_info_begin());
126 WrapperInvoke->setAttributes(NewAttributes);
127 WrapperInvoke->setDebugLoc(Invoke->getDebugLoc());
128 NewCall = WrapperInvoke;
129 } else if (auto Call = dyn_cast(CB)) {
131
133 WrapperCall->setDebugLoc(Call->getDebugLoc());
134 NewCall = WrapperCall;
135 } else {
136 llvm_unreachable("Unexpected coro_await_suspend invocation method");
137 }
138
140 Intrinsic::coro_await_suspend_handle) {
141
142
143 if (auto *Invoke = dyn_cast(CB)) {
144
145 Builder.SetInsertPoint(Invoke->getNormalDest()->getFirstInsertionPt());
146 }
147
151
154 Type::getVoidTy(Ctx), PointerType::getUnqual(Ctx), false);
155 auto *ResumeCall = Builder.CreateCall(ResumeTy, ResumeAddr, {NewCall});
157
158
159
161
162 NewCall = ResumeCall;
163 }
164
167}
168
173}
174
178 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce);
180 return;
181
183}
184
185
186
187
190
191 auto *EndAsync = dyn_cast(End);
192 if (!EndAsync) {
194 return true ;
195 }
196
197 auto *MustTailCallFunc = EndAsync->getMustTailCallFunction();
198 if (!MustTailCallFunc) {
200 return true ;
201 }
202
203
204 auto *CoroEndBlock = End->getParent();
205 auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor();
206 assert(MustTailCallFuncBlock && "Must have a single predecessor block");
207 auto It = MustTailCallFuncBlock->getTerminator()->getIterator();
208 auto *MustTailCall = cast(&*std::prev(It));
209 CoroEndBlock->splice(End->getIterator(), MustTailCallFuncBlock,
210 MustTailCall->getIterator());
211
212
216
217
218 auto *BB = End->getParent();
219 BB->splitBasicBlock(End);
220 BB->getTerminator()->eraseFromParent();
221
222 auto InlineRes = InlineFunction(*MustTailCall, FnInfo);
223 assert(InlineRes.isSuccess() && "Expected inlining to succeed");
224 (void)InlineRes;
225
226
227 return false;
228}
229
230
234
236
237
238 switch (Shape.ABI) {
239
240 case coro::ABI::Switch:
241 assert(!cast(End)->hasResults() &&
242 "switch coroutine should not return any values");
243
244
245 if (!InResume)
246 return;
248 break;
249
250
251 case coro::ABI::Async: {
253 if (!CoroEndBlockNeedsCleanup)
254 return;
255 break;
256 }
257
258
259
260 case coro::ABI::RetconOnce: {
262 auto *CoroEnd = cast(End);
264
265 if (!CoroEnd->hasResults()) {
268 break;
269 }
270
271 auto *CoroResults = CoroEnd->getResults();
272 unsigned NumReturns = CoroResults->numReturns();
273
274 if (auto *RetStructTy = dyn_cast(RetTy)) {
275 assert(RetStructTy->getNumElements() == NumReturns &&
276 "numbers of returns should match resume function singature");
278 unsigned Idx = 0;
279 for (Value *RetValEl : CoroResults->return_values())
282 } else if (NumReturns == 0) {
285 } else {
286 assert(NumReturns == 1);
287 Builder.CreateRet(*CoroResults->retval_begin());
288 }
291 CoroResults->eraseFromParent();
292 break;
293 }
294
295
296
297 case coro::ABI::Retcon: {
298 assert(!cast(End)->hasResults() &&
299 "retcon coroutine should not return any values");
302 auto RetStructTy = dyn_cast(RetTy);
304 cast(RetStructTy ? RetStructTy->getElementType(0) : RetTy);
305
307 if (RetStructTy) {
309 ReturnValue, 0);
310 }
312 break;
313 }
314 }
315
316
317 auto *BB = End->getParent();
318 BB->splitBasicBlock(End);
319 BB->getTerminator()->eraseFromParent();
320}
321
322
323
324
325
326
327
328
329
333 Shape.ABI == coro::ABI::Switch &&
334 "markCoroutineAsDone is only supported for Switch-Resumed ABI for now.");
337 "ResumeFn.addr");
341
342
343
344
345
346
347
348
349
352 assert(cast(Shape.CoroSuspends.back())->isFinal() &&
353 "The final suspend should only live in the last position of "
354 "CoroSuspends.");
358
359 Builder.CreateStore(IndexVal, FinalIndex);
360 }
361}
362
363
368
369 switch (Shape.ABI) {
370
371 case coro::ABI::Switch: {
372
373
374
375
376
377
379 if (!InResume)
380 return;
381 break;
382 }
383
384 case coro::ABI::Async:
385 break;
386
387 case coro::ABI::Retcon:
388 case coro::ABI::RetconOnce:
390 break;
391 }
392
393
395 auto *FromPad = cast(Bundle->Inputs[0]);
396 auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);
397 End->getParent()->splitBasicBlock(End);
398 CleanupRet->getParent()->getTerminator()->eraseFromParent();
399 }
400}
401
404 if (End->isUnwind())
406 else
408
409 auto &Context = End->getContext();
412 End->eraseFromParent();
413}
414
415
416
417
418
419
420
421
422
423
427
429 return;
430
432 auto FinalCaseIt = std::prev(Switch->case_end());
433 BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor();
434 Switch->removeCase(FinalCaseIt);
439
441
442
444 } else {
447 "ResumeFn.addr");
448 auto *Load =
452 }
454 }
455}
456
459 auto *AsyncSuspend = cast(Suspend);
460 auto *StructTy = cast(AsyncSuspend->getType());
461 auto &Context = Suspend->getParent()->getParent()->getContext();
464}
465
467 const Twine &Suffix,
474
477 OrigF.getName() + Suffix);
478
479 M->getFunctionList().insert(InsertBefore, NewF);
480
481 return NewF;
482}
483
484
485
486
487
491
492 auto NewS = VMap[ActiveSuspend];
493 if (NewS->use_empty())
494 return;
495
496
497
499
501 for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()),
502 E = NewF->arg_end();
504 Args.push_back(&*I);
505
506
507
508 if (!isa(NewS->getType())) {
509 assert(Args.size() == 1);
510 NewS->replaceAllUsesWith(Args.front());
511 return;
512 }
513
514
516 auto *EVI = dyn_cast(U.getUser());
517 if (!EVI || EVI->getNumIndices() != 1)
518 continue;
519
520 EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]);
521 EVI->eraseFromParent();
522 }
523
524
525 if (NewS->use_empty())
526 return;
527
528
531 Aggr = Builder.CreateInsertValue(Aggr, Arg, Idx);
532
534}
535
537 Value *SuspendResult;
538
540
541
542
543
544
545
547 SuspendResult = Builder.getInt8(isSwitchDestroyFunction() ? 1 : 0);
548 break;
549
550
552 return;
553
554
555
556
559 return;
560 }
561
563
564 if (CS == ActiveSuspend)
565 continue;
566
567 auto *MappedCS = cast(VMap[CS]);
568 MappedCS->replaceAllUsesWith(SuspendResult);
569 MappedCS->eraseFromParent();
570 }
571}
572
575
576
577 auto *NewCE = cast(VMap[CE]);
579 }
580}
581
585 return;
586 Value *CachedSlot = nullptr;
587 auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * {
588 if (CachedSlot)
589 return CachedSlot;
590
591
592 for (auto &Arg : F.args()) {
593 if (Arg.isSwiftError()) {
594 CachedSlot = &Arg;
595 return &Arg;
596 }
597 }
598
599
601 F.getEntryBlock().getFirstNonPHIOrDbg());
604
605 CachedSlot = Alloca;
606 return Alloca;
607 };
608
610 auto MappedOp = VMap ? cast((*VMap)[Op]) : Op;
612
613
614 Value *MappedResult;
615 if (Op->arg_empty()) {
616 auto ValueTy = Op->getType();
617 auto Slot = getSwiftErrorSlot(ValueTy);
618 MappedResult = Builder.CreateLoad(ValueTy, Slot);
619 } else {
621 auto Value = MappedOp->getArgOperand(0);
623 auto Slot = getSwiftErrorSlot(ValueTy);
625 MappedResult = Slot;
626 }
627
628 MappedOp->replaceAllUsesWith(MappedResult);
629 MappedOp->eraseFromParent();
630 }
631
632
633 if (VMap == nullptr) {
635 }
636}
637
638
639static std::pair<SmallVector<DbgVariableIntrinsic *, 8>,
646 DbgVariableRecords.push_back(&DVR);
647 if (auto *DVI = dyn_cast(&I))
649 }
650 return {Intrinsics, DbgVariableRecords};
651}
652
655}
656
660
661
662 bool UseEntryValue =
668
669
670
672 auto IsUnreachableBlock = [&](BasicBlock *BB) {
674 &DomTree);
675 };
676 auto RemoveOne = [&](auto *DVI) {
677 if (IsUnreachableBlock(DVI->getParent()))
678 DVI->eraseFromParent();
679 else if (isa_and_nonnull(DVI->getVariableLocationOp(0))) {
680
681 unsigned Uses = 0;
682 for (auto *User : DVI->getVariableLocationOp(0)->users())
683 if (auto *I = dyn_cast(User))
684 if (!isa(I) && !IsUnreachableBlock(I->getParent()))
687 DVI->eraseFromParent();
688 }
689 };
690 for_each(Worklist, RemoveOne);
691 for_each(DbgVariableRecords, RemoveOne);
692}
693
695
696
697
698
699
701 auto *OldEntry = &NewF->getEntryBlock();
702 Entry->setName("entry" + Suffix);
703 Entry->moveBefore(OldEntry);
704 Entry->getTerminator()->eraseFromParent();
705
706
707
708
709 assert(Entry->hasOneUse());
710 auto BranchToEntry = cast(Entry->user_back());
711 assert(BranchToEntry->isUnconditional());
712 Builder.SetInsertPoint(BranchToEntry);
713 Builder.CreateUnreachable();
714 BranchToEntry->eraseFromParent();
715
716
717 Builder.SetInsertPoint(Entry);
720
721
722 auto *SwitchBB =
724 Builder.CreateBr(SwitchBB);
725 break;
726 }
730
731
732
734 isa(ActiveSuspend)) ||
737 isa(ActiveSuspend)));
738 auto *MappedCS = cast(VMap[ActiveSuspend]);
739 auto Branch = cast(MappedCS->getNextNode());
740 assert(Branch->isUnconditional());
741 Builder.CreateBr(Branch->getSuccessor(0));
742 break;
743 }
744 }
745
746
747
748 Function *F = OldEntry->getParent();
751 auto *Alloca = dyn_cast(&I);
752 if (!Alloca || I.use_empty())
753 continue;
755 !isa(Alloca->getArraySize()))
756 continue;
757 I.moveBefore(*Entry, Entry->getFirstInsertionPt());
758 }
759}
760
761
763
764
766
768 return &*NewF->arg_begin();
769
770
771
772
773
775 auto *ActiveAsyncSuspend = cast(ActiveSuspend);
776 auto ContextIdx = ActiveAsyncSuspend->getStorageArgumentIndex() & 0xff;
777 auto *CalleeContext = NewF->getArg(ContextIdx);
778 auto *ProjectionFunc =
779 ActiveAsyncSuspend->getAsyncContextProjectionFunction();
780 auto DbgLoc =
781 cast(VMap[ActiveSuspend])->getDebugLoc();
782
783 auto *CallerContext = Builder.CreateCall(ProjectionFunc->getFunctionType(),
784 ProjectionFunc, CalleeContext);
785 CallerContext->setCallingConv(ProjectionFunc->getCallingConv());
786 CallerContext->setDebugLoc(DbgLoc);
787
788 auto &Context = Builder.getContext();
789 auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(
792
795 assert(InlineRes.isSuccess());
796 (void)InlineRes;
797 return FramePtrAddr;
798 }
799
802 Argument *NewStorage = &*NewF->arg_begin();
804
805
807 return NewStorage;
808
809
810 return Builder.CreateLoad(FramePtrTy, NewStorage);
811 }
812 }
814}
815
816
817
818
819
820
821
824 if (!ActiveSuspend)
825 return;
826
828
829
830 if (auto *Branch = dyn_cast_or_null(Successor);
831 Branch && Branch->isUnconditional())
832 Successor = &*Branch->getSuccessor(0)->getFirstNonPHIOrDbg();
833
834
835
838 if ( || DL.getLine() == 0)
839 continue;
840
841 if (SPToUpdate.getFile() == DL->getFile()) {
842 SPToUpdate.setScopeLine(DL.getLine());
843 return;
844 }
845
846 break;
847 }
848
849
851 if (SPToUpdate.getFile() == DL->getFile())
852 SPToUpdate.setScopeLine(DL->getLine());
853}
854
857 Align Alignment, bool NoAlias) {
861
862 if (NoAlias)
864
867 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
868}
869
871 unsigned ParamIndex) {
873 ParamAttrs.addAttribute(Attribute::SwiftAsync);
874 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
875}
876
878 unsigned ParamIndex) {
880 ParamAttrs.addAttribute(Attribute::SwiftSelf);
881 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
882}
883
884
885
888
889
890
891
892
893
895 for (Argument &A : OrigF.args()) {
897 VMap[&A] = DummyArgs.back();
898 }
899
901
902
903
904 auto savedVisibility = NewF->getVisibility();
905 auto savedUnnamedAddr = NewF->getUnnamedAddr();
906 auto savedDLLStorageClass = NewF->getDLLStorageClass();
907
908
909
910
911 auto savedLinkage = NewF->getLinkage();
913
916 &CommonDebugInfo);
918 nullptr, nullptr, &CommonDebugInfo);
919
920 auto &Context = NewF->getContext();
921
923 assert(SP != OrigF.getSubprogram() && SP->isDistinct());
925
926
927
928
929
930
931
932
933
934 if (SP->getUnit() &&
935 SP->getUnit()->getSourceLanguage() == dwarf::DW_LANG_Swift) {
936 SP->replaceLinkageName(MDString::get(Context, NewF->getName()));
937 if (auto *Decl = SP->getDeclaration()) {
939 Decl->getContext(), Decl->getScope(), Decl->getName(),
940 NewF->getName(), Decl->getFile(), Decl->getLine(), Decl->getType(),
941 Decl->getScopeLine(), Decl->getContainingType(),
942 Decl->getVirtualIndex(), Decl->getThisAdjustment(),
943 Decl->getFlags(), Decl->getSPFlags(), Decl->getUnit(),
944 Decl->getTemplateParams(), nullptr, Decl->getRetainedNodes(),
945 Decl->getThrownTypes(), Decl->getAnnotations(),
946 Decl->getTargetFuncName());
947 SP->replaceDeclaration(NewDecl);
948 }
949 }
950 }
951
952 NewF->setLinkage(savedLinkage);
953 NewF->setVisibility(savedVisibility);
954 NewF->setUnnamedAddr(savedUnnamedAddr);
955 NewF->setDLLStorageClass(savedDLLStorageClass);
956
957
958
960 NewF->hasMetadata(LLVMContext::MD_func_sanitize))
961 NewF->eraseMetadata(LLVMContext::MD_func_sanitize);
962
963
964 auto OrigAttrs = NewF->getAttributes();
966
969
970
971 NewAttrs = NewAttrs.addFnAttributes(
972 Context, AttrBuilder(Context, OrigAttrs.getFnAttrs()));
973
976 break;
978 auto *ActiveAsyncSuspend = cast(ActiveSuspend);
980 Attribute::SwiftAsync)) {
981 uint32_t ArgAttributeIndices =
982 ActiveAsyncSuspend->getStorageArgumentIndex();
983 auto ContextArgIndex = ArgAttributeIndices & 0xff;
985
986
987
988 auto SwiftSelfIndex = ArgAttributeIndices >> 8;
989 if (SwiftSelfIndex)
991 }
992
993
994 auto FnAttrs = OrigF.getAttributes().getFnAttrs();
995 NewAttrs = NewAttrs.addFnAttributes(Context, AttrBuilder(Context, FnAttrs));
996 break;
997 }
1000
1001
1003
1004
1008 true);
1009
1010 break;
1011 }
1012
1014
1015
1016
1017
1020
1023 break;
1024
1025
1026
1027
1029 break;
1030
1031
1032
1033
1035 break;
1036 }
1037
1038 NewF->setAttributes(NewAttrs);
1040
1041
1042 replaceEntryBlock();
1043
1044
1046 ResumeCall = cast(VMap[ResumeCall]);
1048
1049
1051 }
1052
1053
1054
1058 Builder.CreateRetVoid();
1060 }
1061
1062 Builder.SetInsertPoint(&NewF->getEntryBlock().front());
1063 NewFramePtr = deriveNewFramePointer();
1064
1065
1067 NewFramePtr->takeName(OldFramePtr);
1069
1070
1071 auto *NewVFrame = Builder.CreateBitCast(
1074 if (OldVFrame != NewVFrame)
1076
1077
1078
1079 for (Instruction *DummyArg : DummyArgs) {
1080 DummyArg->replaceAllUsesWith(PoisonValue::get(DummyArg->getType()));
1081 DummyArg->deleteValue();
1082 }
1083
1086
1087
1088
1090 handleFinalSuspend();
1091 break;
1095
1096
1097 assert(ActiveSuspend != nullptr &&
1098 "no active suspend when lowering a continuation-style coroutine");
1099 replaceRetconOrAsyncSuspendUses();
1100 break;
1101 }
1102
1103
1104 replaceCoroSuspends();
1105
1106
1108
1109
1110 replaceCoroEnds();
1111
1112
1114}
1115
1117
1119 ActiveSuspend);
1120
1121
1123
1124
1125
1128}
1129
1132
1133 auto *FuncPtrStruct = cast(
1135 auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0);
1136 auto *OrigContextSize = FuncPtrStruct->getOperand(1);
1137 auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(),
1140 FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize);
1141
1143}
1144
1146
1147 auto *SizeIntrin = Shape.CoroSizes.back();
1148 Module *M = SizeIntrin->getModule();
1150 return DL.getTypeAllocSize(Shape.FrameTy);
1151}
1152
1156
1161 }
1162
1164 return;
1165
1166
1167 auto *SizeIntrin = Shape.CoroSizes.back();
1168 auto *SizeConstant =
1170
1174 }
1175}
1176
1179
1180#ifndef NDEBUG
1181
1182
1183
1186#endif
1187}
1188
1189
1190
1192 auto *CoroBegin = Shape.CoroBegin;
1193 switch (Shape.ABI) {
1198 if (AllocInst) {
1202 AllocInst->replaceAllUsesWith(Builder.getFalse());
1203 AllocInst->eraseFromParent();
1204 CoroBegin->replaceAllUsesWith(Frame);
1205 } else {
1206 CoroBegin->replaceAllUsesWith(CoroBegin->getMem());
1207 }
1208
1209 break;
1210 }
1214 CoroBegin->replaceAllUsesWith(PoisonValue::get(CoroBegin->getType()));
1215 break;
1216 }
1217
1218 CoroBegin->eraseFromParent();
1220}
1221
1222
1223
1224
1227
1228 if (isa(I))
1229 continue;
1230
1231 if (isa(I))
1232 return true;
1233 }
1234 return false;
1235}
1236
1240
1241 Set.insert(SaveBB);
1243
1244
1245
1246
1247 while (!Worklist.empty()) {
1249 Set.insert(BB);
1251 if (!Set.contains(Pred))
1253 }
1254
1255
1256 Set.erase(SaveBB);
1257 Set.erase(ResDesBB);
1258
1259 for (auto *BB : Set)
1261 return true;
1262
1263 return false;
1264}
1265
1267 auto *SaveBB = Save->getParent();
1268 auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();
1271
1272 if (SaveBB == ResumeOrDestroyBB)
1274
1275
1277 return true;
1278
1279
1281 {ResumeOrDestroyBB->getFirstNonPHIIt(), ResumeOrDestroyIt}))
1282 return true;
1283
1284
1286 return true;
1287
1288 return false;
1289}
1290
1291
1292
1296 if (!Prev) {
1297 auto *Pred = Suspend->getParent()->getSinglePredecessor();
1298 if (!Pred)
1299 return false;
1300 Prev = Pred->getTerminator();
1301 }
1302
1303 CallBase *CB = dyn_cast(Prev);
1304 if (!CB)
1305 return false;
1306
1308
1309
1310 auto *SubFn = dyn_cast(Callee);
1311 if (!SubFn)
1312 return false;
1313
1314
1315 if (SubFn->getFrame() != CoroBegin)
1316 return false;
1317
1318
1319
1320
1323 return false;
1324
1325
1326
1329 Save->eraseFromParent();
1330
1331
1332 if (auto *Invoke = dyn_cast(CB)) {
1334 }
1335
1336
1339
1340
1341 if (CalledValue != SubFn && CalledValue->user_empty())
1342 if (auto *I = dyn_cast(CalledValue))
1343 I->eraseFromParent();
1344
1345
1346 if (SubFn->user_empty())
1347 SubFn->eraseFromParent();
1348
1349 return true;
1350}
1351
1352
1354
1356 return;
1357
1359 size_t I = 0, N = S.size();
1360 if (N == 0)
1361 return;
1362
1363 size_t ChangedFinalIndex = std::numeric_limits<size_t>::max();
1364 while (true) {
1365 auto SI = cast(S[I]);
1366
1367
1370 break;
1371
1373
1374 if (cast(S[I])->isFinal()) {
1376 ChangedFinalIndex = I;
1377 }
1378
1379 continue;
1380 }
1382 break;
1383 }
1384 S.resize(N);
1385
1386
1387
1388 if (ChangedFinalIndex < N) {
1389 assert(cast(S[ChangedFinalIndex])->isFinal());
1390 std::swap(S[ChangedFinalIndex], S.back());
1391 }
1392}
1393
1394namespace {
1395
1396struct SwitchCoroutineSplitter {
1400 assert(Shape.ABI == coro::ABI::Switch);
1401
1402 MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F)};
1403
1404
1405
1406
1407 createResumeEntryBlock(F, Shape);
1409 F, ".resume", Shape, coro::CloneKind::SwitchResume, TTI,
1410 CommonDebugInfo);
1412 F, ".destroy", Shape, coro::CloneKind::SwitchUnwind, TTI,
1413 CommonDebugInfo);
1415 F, ".cleanup", Shape, coro::CloneKind::SwitchCleanup, TTI,
1416 CommonDebugInfo);
1417
1421
1422
1423 updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);
1424
1429
1430
1431
1432
1433 setCoroInfo(F, Shape, Clones);
1434 }
1435
1436
1437
1438
1439
1440
1441
1442
1445 assert(Shape.ABI == coro::ABI::Switch);
1446 auto *OrigFnTy = F.getFunctionType();
1447 auto OldParams = OrigFnTy->params();
1448
1450 NewParams.reserve(OldParams.size() + 1);
1451 NewParams.append(OldParams.begin(), OldParams.end());
1453
1454 auto *NewFnTy = FunctionType::get(OrigFnTy->getReturnType(), NewParams,
1455 OrigFnTy->isVarArg());
1457 Function::Create(NewFnTy, F.getLinkage(), F.getName() + ".noalloc");
1458
1460 unsigned int Idx = 0;
1461 for (const auto &I : F.args()) {
1463 }
1464
1465
1466 auto FrameIdx = NoAllocF->arg_size() - 1;
1469 CloneFunctionChangeType::LocalChangesOnly, Returns);
1470
1472 auto *NewCoroBegin =
1473 cast_if_present(VMap[Shape.CoroBegin]);
1474 auto *NewCoroId = cast(NewCoroBegin->getId());
1477 NewCoroBegin->replaceAllUsesWith(NoAllocF->getArg(FrameIdx));
1478 NewCoroBegin->eraseFromParent();
1479 }
1480
1482 M->getFunctionList().insert(M->end(), NoAllocF);
1483
1486
1487
1490 false);
1491
1493
1495
1496
1497 setCoroInfo(F, Shape, Clones);
1498
1499
1500
1502 return NoAllocF;
1503 }
1504
1505private:
1506
1507
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1522
1525 auto *FrameTy = Shape.FrameTy;
1526 auto *GepIndex = Builder.CreateStructGEP(
1528 auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");
1530 Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
1532
1533 size_t SuspendIndex = 0;
1535 auto *S = cast(AnyS);
1537
1538
1539
1540
1541 auto *Save = S->getCoroSave();
1542 Builder.SetInsertPoint(Save);
1543 if (S->isFinal()) {
1544
1545
1547 } else {
1548 auto *GepIndex = Builder.CreateStructGEP(
1550 Builder.CreateStore(IndexVal, GepIndex);
1551 }
1552
1554 Save->eraseFromParent();
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579 auto *SuspendBB = S->getParent();
1580 auto *ResumeBB =
1581 SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex));
1582 auto *LandingBB = ResumeBB->splitBasicBlock(
1583 S->getNextNode(), ResumeBB->getName() + Twine(".landing"));
1584 Switch->addCase(IndexVal, ResumeBB);
1585
1586 cast(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);
1588 PN->insertBefore(LandingBB->begin());
1589 S->replaceAllUsesWith(PN);
1590 PN->addIncoming(Builder.getInt8(-1), SuspendBB);
1591 PN->addIncoming(S, ResumeBB);
1592
1593 ++SuspendIndex;
1594 }
1595
1596 Builder.SetInsertPoint(UnreachBB);
1597 Builder.CreateUnreachable();
1598
1600 }
1601
1602
1606
1607 auto *ResumeAddr = Builder.CreateStructGEP(
1609 "resume.addr");
1610 Builder.CreateStore(ResumeFn, ResumeAddr);
1611
1612 Value *DestroyOrCleanupFn = DestroyFn;
1613
1616
1617
1618 DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);
1619 }
1620
1621 auto *DestroyAddr = Builder.CreateStructGEP(
1623 "destroy.addr");
1624 Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);
1625 }
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1642
1643
1648 auto *ArrTy = ArrayType::get(Part->getType(), Args.size());
1649
1651 auto *GV = new GlobalVariable(*M, ConstVal->getType(), true,
1652 GlobalVariable::PrivateLinkage, ConstVal,
1653 F.getName() + Twine(".resumers"));
1654
1655
1659 }
1660};
1661
1662}
1663
1667 auto &Context = Suspend->getParent()->getParent()->getContext();
1669
1673 ResumeIntrinsic->eraseFromParent();
1676}
1677
1678
1682 size_t ArgIdx = 0;
1683 for (auto *paramTy : FnTy->params()) {
1685 if (paramTy != FnArgs[ArgIdx]->getType())
1688 else
1689 CallArgs.push_back(FnArgs[ArgIdx]);
1690 ++ArgIdx;
1691 }
1692}
1693
1699
1700
1703
1704 auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs);
1705
1708 }
1709 TailCall->setDebugLoc(Loc);
1710 TailCall->setCallingConv(MustTailCallFn->getCallingConv());
1711 return TailCall;
1712}
1713
1719
1720
1721 F.removeFnAttr(Attribute::NoReturn);
1722 F.removeRetAttr(Attribute::NoAlias);
1723 F.removeRetAttr(Attribute::NonNull);
1724
1725 auto &Context = F.getContext();
1727
1730
1731 auto *FramePtr = Id->getStorage();
1735 "async.ctx.frameptr");
1736
1737
1738 {
1739
1743 }
1744
1745
1746 auto NextF = std::next(F.getIterator());
1747
1748
1751 auto *Suspend = cast(CS);
1752
1753
1754 auto ResumeNameSuffix = ".resume.";
1755 auto ProjectionFunctionName =
1756 Suspend->getAsyncContextProjectionFunction()->getName();
1757 bool UseSwiftMangling = false;
1758 if (ProjectionFunctionName == "__swift_async_resume_project_context") {
1759 ResumeNameSuffix = "TQ";
1760 UseSwiftMangling = true;
1761 } else if (ProjectionFunctionName == "__swift_async_resume_get_context") {
1762 ResumeNameSuffix = "TY";
1763 UseSwiftMangling = true;
1764 }
1767 UseSwiftMangling ? ResumeNameSuffix + Twine(Idx) + "_"
1768 : ResumeNameSuffix + Twine(Idx),
1769 NextF, Suspend);
1771
1772
1773
1774 auto *SuspendBB = Suspend->getParent();
1775 auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1776 auto *Branch = cast(SuspendBB->getTerminator());
1777
1778
1779 auto *ReturnBB =
1781 Branch->setSuccessor(0, ReturnBB);
1782
1784
1785
1786 auto *Fn = Suspend->getMustTailCallFunction();
1791 FnArgs, Builder);
1795
1796
1798 }
1799
1801
1802 MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F)};
1803
1805 auto *Suspend = CS;
1806 auto *Clone = Clones[Idx];
1807
1809 Suspend, TTI, CommonDebugInfo);
1810 }
1811}
1812
1818
1819
1820
1821 F.removeFnAttr(Attribute::NoReturn);
1822 F.removeRetAttr(Attribute::NoAlias);
1823 F.removeRetAttr(Attribute::NonNull);
1824
1825
1827 Value *RawFramePtr;
1829 RawFramePtr = Id->getStorage();
1830 } else {
1832
1833
1836
1837
1838
1839
1841 RawFramePtr =
1843
1844
1845 Builder.CreateStore(RawFramePtr, Id->getStorage());
1846 }
1847
1848
1849 {
1850
1854 }
1855
1856
1858 PHINode *ContinuationPhi = nullptr;
1860
1861
1862 auto NextF = std::next(F.getIterator());
1863
1864
1867 auto Suspend = cast(CS);
1868
1869
1873
1874
1875
1876 auto SuspendBB = Suspend->getParent();
1877 auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1878 auto Branch = cast(SuspendBB->getTerminator());
1879
1880
1881 if (!ReturnBB) {
1882
1883 ReturnBB =
1886
1888
1889
1890 ContinuationPhi =
1892
1893
1895
1896
1900
1901
1902 auto RetTy = F.getReturnType();
1903
1904
1905
1906
1907 auto CastedContinuationTy =
1908 (ReturnPHIs.empty() ? RetTy : RetTy->getStructElementType(0));
1909 auto *CastedContinuation =
1910 Builder.CreateBitCast(ContinuationPhi, CastedContinuationTy);
1911
1912 Value *RetV = CastedContinuation;
1913 if (!ReturnPHIs.empty()) {
1914 auto ValueIdx = 0;
1916 RetV = Builder.CreateInsertValue(RetV, CastedContinuation, ValueIdx++);
1917
1918 for (auto Phi : ReturnPHIs)
1920 }
1921
1923 }
1924
1925
1926 Branch->setSuccessor(0, ReturnBB);
1927 assert(ContinuationPhi);
1929 for (auto [Phi, VUse] :
1931 Phi->addIncoming(VUse, SuspendBB);
1932 }
1933
1935
1936 MetadataSetTy CommonDebugInfo{collectCommonDebugInfo(F)};
1937
1939 auto Suspend = CS;
1940 auto Clone = Clones[Idx];
1941
1943 Suspend, TTI, CommonDebugInfo);
1944 }
1945}
1946
1947namespace {
1950
1951public:
1952 PrettyStackTraceFunction(Function &F) : F(F) {}
1954 OS << "While splitting coroutine ";
1955 F.printAsOperand(OS, false, F.getParent());
1956 OS << "\n";
1957 }
1958};
1959}
1960
1961
1966 }
1967 } else {
1969 auto &Context = End->getContext();
1971 End->eraseFromParent();
1972 }
1973 }
1974}
1975
1977 for (auto *U : F.users()) {
1978 if (auto *CB = dyn_cast(U)) {
1979 auto *Caller = CB->getFunction();
1980 if (Caller && Caller->isPresplitCoroutine() &&
1981 CB->hasFnAttr(llvm::Attribute::CoroElideSafe))
1982 return true;
1983 }
1984 }
1985 return false;
1986}
1987
1991 SwitchCoroutineSplitter::split(F, Shape, Clones, TTI);
1992}
1993
1996 bool OptimizeFrame) {
1997 PrettyStackTraceFunction prettyStackTrace(F);
1998
1999 auto &Shape = ABI.Shape;
2001
2003
2005
2007 ABI.buildCoroutineFrame(OptimizeFrame);
2009
2010 bool isNoSuspendCoroutine = Shape.CoroSuspends.empty();
2011
2012 bool shouldCreateNoAllocVariant =
2015
2016
2017
2018 if (isNoSuspendCoroutine) {
2020 } else {
2021 ABI.splitCoroutine(F, Shape, Clones, TTI);
2022 }
2023
2024
2025
2027
2028
2029
2030
2033 for (auto *DDI : DbgInsts)
2037
2039
2040 if (shouldCreateNoAllocVariant)
2041 SwitchCoroutineSplitter::createNoAllocVariant(F, Shape, Clones);
2042}
2043
2049
2050 auto *CurrentSCC = &C;
2051 if (!Clones.empty()) {
2052 switch (Shape.ABI) {
2054
2055
2056 for (Function *Clone : Clones)
2058 break;
2062
2063
2064 if (!Clones.empty())
2066 break;
2067 }
2068
2069
2071 UR, FAM);
2072 }
2073
2074
2075
2078 AM, UR, FAM);
2079 return *CurrentSCC;
2080}
2081
2082
2085 auto CastFn = Prepare->getArgOperand(0);
2087
2088
2089
2090
2091
2092
2093
2095
2096 auto *Cast = dyn_cast(U.getUser());
2097 if (!Cast || Cast->getType() != Fn->getType())
2098 continue;
2099
2100
2101 Cast->replaceAllUsesWith(Fn);
2102 Cast->eraseFromParent();
2103 }
2104
2105
2106
2109
2110
2111 while (auto *Cast = dyn_cast(CastFn)) {
2112 if (!Cast->use_empty())
2113 break;
2114 CastFn = Cast->getOperand(0);
2115 Cast->eraseFromParent();
2116 }
2117}
2118
2121 bool Changed = false;
2123
2124 auto *Prepare = cast(P.getUser());
2126 Changed = true;
2127 }
2128
2129 return Changed;
2130}
2131
2135 auto *PrepareFn = M.getFunction(Name);
2136 if (PrepareFn && !PrepareFn->use_empty())
2138}
2139
2140static std::unique_ptrcoro::BaseABI
2142 std::function<bool(Instruction &)> IsMatCallback,
2146 if (CustomABI >= GenCustomABIs.size())
2147 llvm_unreachable("Custom ABI not found amoung those specified");
2148 return GenCustomABIs[CustomABI](F, S);
2149 }
2150
2151 switch (S.ABI) {
2153 return std::make_uniquecoro::SwitchABI(F, S, IsMatCallback);
2155 return std::make_uniquecoro::AsyncABI(F, S, IsMatCallback);
2157 return std::make_uniquecoro::AnyRetconABI(F, S, IsMatCallback);
2159 return std::make_uniquecoro::AnyRetconABI(F, S, IsMatCallback);
2160 }
2162}
2163
2165 : CreateAndInitABI([](Function &F, coro::Shape &S) {
2166 std::unique_ptrcoro::BaseABI ABI =
2168 ABI->init();
2169 return ABI;
2170 }),
2171 OptimizeFrame(OptimizeFrame) {}
2172
2175 : CreateAndInitABI([=](Function &F, coro::Shape &S) {
2176 std::unique_ptrcoro::BaseABI ABI =
2178 ABI->init();
2179 return ABI;
2180 }),
2181 OptimizeFrame(OptimizeFrame) {}
2182
2183
2184
2186 bool OptimizeFrame)
2187 : CreateAndInitABI([=](Function &F, coro::Shape &S) {
2188 std::unique_ptrcoro::BaseABI ABI =
2190 ABI->init();
2191 return ABI;
2192 }),
2193 OptimizeFrame(OptimizeFrame) {}
2194
2195
2196
2198 std::function<bool(Instruction &)> IsMatCallback,
2200 : CreateAndInitABI([=](Function &F, coro::Shape &S) {
2201 std::unique_ptrcoro::BaseABI ABI =
2202 CreateNewABI(F, S, IsMatCallback, GenCustomABIs);
2203 ABI->init();
2204 return ABI;
2205 }),
2206 OptimizeFrame(OptimizeFrame) {}
2207
2211
2212
2213
2214 Module &M = *C.begin()->getFunction().getParent();
2215 auto &FAM =
2217
2218
2222
2223
2226 if (N.getFunction().isPresplitCoroutine())
2228
2229 if (Coroutines.empty() && PrepareFns.empty())
2231
2232 auto *CurrentSCC = &C;
2233
2236 LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()
2237 << "\n");
2238
2239
2240
2241
2243
2246 continue;
2247
2248 F.setSplittedCoroutine();
2249
2250 std::unique_ptrcoro::BaseABI ABI = CreateAndInitABI(F, Shape);
2251
2256 *N, Shape, Clones, *CurrentSCC, CG, AM, UR, FAM);
2257
2259 ORE.emit([&]() {
2261 << "Split '" << ore::NV("function", F.getName())
2262 << "' (frame_size=" << ore::NV("frame_size", Shape.FrameSize)
2264 });
2265
2267
2268 UR.CWorklist.insert(CurrentSCC);
2269 for (Function *Clone : Clones)
2271 }
2272 }
2273
2274 for (auto *PrepareFn : PrepareFns) {
2276 }
2277
2279}
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
AMDGPU Lower Kernel Arguments
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file provides interfaces used to manipulate a call graph, regardless if it is a "old style" Call...
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy)
static std::pair< SmallVector< DbgVariableIntrinsic *, 8 >, SmallVector< DbgVariableRecord * > > collectDbgVariableIntrinsics(Function &F)
Returns all DbgVariableIntrinsic in F.
static LazyCallGraph::SCC & updateCallGraphAfterCoroutineSplit(LazyCallGraph::Node &N, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, LazyCallGraph::SCC &C, LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)
static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
static void maybeFreeRetconStorage(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG)
static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB)
static Function * createCloneDeclaration(Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)
Remove calls to llvm coro end in the original static function void removeCoroEndsFromRampFunction(const coro::Shape &Shape)
static FunctionType * getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend)
static void updateScopeLine(Instruction *ActiveSuspend, DISubprogram &SPToUpdate)
Adjust the scope line of the funclet to the first line number after the suspend point.
static void addPrepareFunction(const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)
static void simplifySuspendPoints(coro::Shape &Shape)
static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment, bool NoAlias)
static bool hasSafeElideCaller(Function &F)
static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)
static void replaceFallthroughCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace a non-unwind call to llvm.coro.end.
static void replaceFrameSizeAndAlignment(coro::Shape &Shape)
static std::unique_ptr< coro::BaseABI > CreateNewABI(Function &F, coro::Shape &S, std::function< bool(Instruction &)> IsMatCallback, const SmallVector< CoroSplitPass::BaseABITy > GenCustomABIs)
static bool replaceCoroEndAsync(AnyCoroEndInst *End)
Replace an llvm.coro.end.async.
static void doSplitCoroutine(Function &F, SmallVectorImpl< Function * > &Clones, coro::BaseABI &ABI, TargetTransformInfo &TTI, bool OptimizeFrame)
static bool hasCallsInBlockBetween(iterator_range< BasicBlock::iterator > R)
Replace a call to llvm coro prepare static retcon void replacePrepare(CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)
static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace an unwind call to llvm.coro.end.
static bool simplifySuspendPoint(CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)
static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr)
static void updateAsyncFuncPointerContextSize(coro::Shape &Shape)
static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
static void lowerAwaitSuspend(IRBuilder<> &Builder, CoroAwaitSuspendInst *CB, coro::Shape &Shape)
static void lowerAwaitSuspends(Function &F, coro::Shape &Shape)
static void handleNoSuspendCoroutine(coro::Shape &Shape)
static void postSplitCleanup(Function &F)
static TypeSize getFrameSizeForShape(coro::Shape &Shape)
Coerce the arguments in p FnArgs according to p FnTy in p static CallArgs void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > &CallArgs)
static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend, Value *Continuation)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
This file contains constants used for implementing Dwarf debug support.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
Implements a lazy call graph analysis and related passes for the new pass manager.
FunctionAnalysisManager FAM
This file provides a priority worklist.
const SmallVectorImpl< MachineOperand > & Cond
Remove Loads Into Fake Uses
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
This pass exposes codegen information to IR-level passes.
static const unsigned FramePtr
void setSwiftError(bool V)
Specify whether this alloca is used to represent a swifterror.
void setAlignment(Align Align)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
CoroAllocInst * getCoroAlloc()
Align getStorageAlignment() const
uint64_t getStorageSize() const
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
size_t size() const
size - Get the array size.
AttrBuilder & addAlignmentAttr(MaybeAlign Align)
This turns an alignment into the form used internally in Attribute.
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
AttrBuilder & addDereferenceableAttr(uint64_t Bytes)
This turns the number of dereferenceable bytes into the form used internally in Attribute.
AttributeList removeParamAttributes(LLVMContext &C, unsigned ArgNo, const AttributeMask &AttrsToRemove) const
Remove the specified attribute at the specified arg index from this attribute list.
LLVM Basic Block Representation.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
const Function * getParent() const
Return the enclosing method, or null if none.
InstListType::iterator iterator
Instruction iterators...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the attributes for this call.
Value * getArgOperand(unsigned i) const
AttributeList getAttributes() const
Return the attributes for this call.
The basic data container for the call graph of a Module of IR.
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
This is the shared class of boolean and integer constants.
static ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getFalse(LLVMContext &Context)
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
This represents the llvm.coro.align instruction.
This represents the llvm.coro.alloc instruction.
This represents the llvm.coro.await.suspend.{void,bool,handle} instructions.
Value * getAwaiter() const
Function * getWrapperFunction() const
This class represents the llvm.coro.begin or llvm.coro.begin.custom.abi instructions.
AnyCoroIdInst * getId() const
bool hasCustomABI() const
This represents the llvm.coro.id instruction.
void setInfo(Constant *C)
This represents the llvm.coro.size instruction.
This represents the llvm.coro.suspend.async instruction.
CoroAsyncResumeInst * getResumeFunction() const
This represents the llvm.coro.suspend instruction.
CoroSaveInst * getCoroSave() const
DISubprogram * getSubprogram() const
Get the subprogram for this scope.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
This is the common base class for debug info intrinsics for variables.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
Utility to find all debug info in a module.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
This class represents a freeze function that returns random concrete value if an operand is either a ...
A proxy from a FunctionAnalysisManager to an SCC.
Class to represent function types.
Type * getReturnType() const
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
void setAttributes(AttributeList Attrs)
Set the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool isCoroOnlyDestroyWhenComplete() const
Argument * getArg(unsigned i) const
void setLinkage(LinkageTypes LT)
Module * getParent()
Get the module that this global value is contained inside of...
PointerType * getType() const
Global values are always pointers.
@ InternalLinkage
Rename collisions when linking (static functions).
@ ExternalLinkage
Externally visible function.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void setInitializer(Constant *InitVal)
setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
InvokeInst * CreateInvoke(FunctionType *Ty, Value *Callee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef< Value * > Args, ArrayRef< OperandBundleDef > OpBundles, const Twine &Name="")
Create an invoke instruction.
BasicBlock::iterator GetInsertPoint() const
Value * CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx, const Twine &Name="")
Value * CreateConstInBoundsGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
CleanupReturnInst * CreateCleanupRet(CleanupPadInst *CleanupPad, BasicBlock *UnwindBB=nullptr)
ReturnInst * CreateRet(Value *V)
Create a 'ret ' instruction.
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
LLVMContext & getContext() const
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
ConstantInt * getFalse()
Get the constant value for i1 false.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
BranchInst * CreateBr(BasicBlock *Dest)
Create an unconditional 'br label X' instruction.
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Instruction * getNextNonDebugInstruction(bool SkipPseudoOp=false) const
Return a pointer to the next non-debug instruction in the same basic block as 'this',...
This is an important class for using LLVM in a threaded context.
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
void addSplitFunction(Function &OriginalFunction, Function &NewFunction)
Add a new function split/outlined from an existing function.
void addSplitRefRecursiveFunctions(Function &OriginalFunction, ArrayRef< Function * > NewFunctions)
Add new ref-recursive functions split/outlined from an existing function.
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
SCC * lookupSCC(Node &N) const
Lookup a function's SCC in the graph.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static MDString * get(LLVMContext &Context, StringRef Str)
A Module instance is used to store all the information related to an LLVM module.
FunctionListType::iterator iterator
The Function iterators.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PrettyStackTraceEntry - This class is used to represent a frame of the "pretty" stack trace that is d...
Return a value (possibly void), from a function.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
Analysis pass providing the TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
bool supportsTailCallFor(const CallBase *CB) const
If target supports tail call on CB.
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Value handle that tracks a Value across RAUW.
ValueTy * getValPtr() const
Triple - Helper class for working with autoconf configuration names.
bool isArch64Bit() const
Test whether the architecture is 64-bit.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
void setOperand(unsigned i, Value *Val)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
LLVMContext & getContext() const
All values hold a context through their type.
iterator_range< use_iterator > uses()
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI) override
void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI) override
static Function * createClone(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, Function *NewF, AnyCoroSuspendInst *ActiveSuspend, TargetTransformInfo &TTI, const MetadataSetTy &CommonDebugInfo)
Create a clone for a continuation lowering.
void replaceSwiftErrorOps()
Value * deriveNewFramePointer()
Derive the value of the new frame pointer.
void replaceCoroSuspends()
void handleFinalSuspend()
bool isSwitchDestroyFunction()
void replaceRetconOrAsyncSuspendUses()
Replace uses of the active llvm.coro.suspend.retcon/async call with the arguments to the continuation...
virtual void create()
Clone the body of the original function into a resume function of some sort.
void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI) override
void create() override
Clone the body of the original function into a resume function of some sort.
static Function * createClone(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, CloneKind FKind, TargetTransformInfo &TTI, const MetadataSetTy &CommonDebugInfo)
Create a clone for a switch lowering.
const ParentTy * getParent() const
self_iterator getIterator()
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
void salvageDebugInfo(SmallDenseMap< Argument *, AllocaInst *, 4 > &ArgToAllocaMap, DbgVariableIntrinsic &DVI, bool IsEntryPoint)
Attempts to rewrite the location operand of debug intrinsics in terms of the coroutine frame pointer,...
@ Async
The "async continuation" lowering, where each suspend point creates a single continuation function.
@ RetconOnce
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
@ Retcon
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
void suppressCoroAllocs(CoroIdInst *CoroId)
Replaces all @llvm.coro.alloc intrinsics calls associated with a given call @llvm....
void normalizeCoroutine(Function &F, coro::Shape &Shape, TargetTransformInfo &TTI)
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, TargetTransformInfo &TTI, ArrayRef< Value * > Arguments, IRBuilder<> &)
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
bool isTriviallyMaterializable(Instruction &I)
@ SwitchCleanup
The shared cleanup function for a switch lowering.
@ Continuation
An individual continuation function.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
void CloneFunctionAttributesInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, bool ModuleLevelChanges, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Clone OldFunc's attributes into NewFunc, transforming values based on the mappings in VMap.
UnaryFunction for_each(R &&Range, UnaryFunction F)
Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.
detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)
zip iterator that assumes that all iteratees have the same length.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a function pass.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
MetadataSetTy FindDebugInfoToIdentityMap(CloneFunctionChangeType Changes, DebugInfoFinder &DIFinder, DISubprogram *SPClonedWithinModule)
Based on Changes and DIFinder return debug info that needs to be identity mapped during Metadata clon...
unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void CloneFunctionMetadataInto(Function &NewFunc, const Function &OldFunc, ValueToValueMapTy &VMap, RemapFlags RemapFlag, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr, const MetadataSetTy *IdentityMD=nullptr)
Clone OldFunc's metadata into NewFunc.
DWARFExpression::Operation Op
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.
auto predecessors(const MachineBasicBlock *BB)
DISubprogram * CollectDebugInfoForCloning(const Function &F, CloneFunctionChangeType Changes, DebugInfoFinder &DIFinder)
Collect debug information such as types, compile units, and other subprograms that are reachable from...
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
void CloneFunctionBodyInto(Function &NewFunc, const Function &OldFunc, ValueToValueMapTy &VMap, RemapFlags RemapFlag, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr, const MetadataSetTy *IdentityMD=nullptr)
Clone OldFunc's body into NewFunc.
bool isPotentiallyReachable(const Instruction *From, const Instruction *To, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet=nullptr, const DominatorTree *DT=nullptr, const LoopInfo *LI=nullptr)
Determine whether instruction 'To' is reachable from 'From', without passing through any blocks in Ex...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
SmallPriorityWorklist< LazyCallGraph::SCC *, 1 > & CWorklist
Worklist of the SCCs queued for processing.
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
CoroSplitPass(bool OptimizeFrame=false)
BaseABITy CreateAndInitABI
CallInst * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
GlobalVariable * AsyncFuncPointer
bool IsFrameInlineInStorage
Function * ResumePrototype
SwitchInst * ResumeSwitch
BasicBlock * ResumeEntryBlock
SmallVector< CallInst *, 2 > SymmetricTransfers
SmallVector< CoroAwaitSuspendInst *, 4 > CoroAwaitSuspends
AsyncLoweringStorage AsyncLowering
FunctionType * getResumeFunctionType() const
IntegerType * getIndexType() const
AnyCoroIdRetconInst * getRetconCoroId() const
PointerType * getSwitchResumePointerType() const
CoroIdInst * getSwitchCoroId() const
SmallVector< CoroSizeInst *, 2 > CoroSizes
CallingConv::ID getResumeFunctionCC() const
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
ConstantInt * getIndex(uint64_t Value) const
SwitchLoweringStorage SwitchLowering
CoroBeginInst * CoroBegin
BasicBlock::iterator getInsertPtAfterFramePtr() const
ArrayRef< Type * > getRetconResultTypes() const
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
RetconLoweringStorage RetconLowering
SmallVector< CoroAlignInst *, 2 > CoroAligns
CoroIdAsyncInst * getAsyncCoroId() const
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
SmallVector< CallInst *, 2 > SwiftErrorOps
BasicBlock * AllocaSpillBlock
unsigned getSwitchIndexField() const