LLVM: lib/Transforms/Coroutines/Coroutines.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
33#include
34#include
35#include
36
37using namespace llvm;
38
39
46
47
48
49
50
51
55 auto *Fn =
57
60 "makeSubFnCall: Index value out of range");
62}
63
64
65
66
68 Intrinsic::coro_alloc,
69 Intrinsic::coro_async_context_alloc,
70 Intrinsic::coro_async_context_dealloc,
71 Intrinsic::coro_async_resume,
72 Intrinsic::coro_async_size_replace,
73 Intrinsic::coro_await_suspend_bool,
74 Intrinsic::coro_await_suspend_handle,
75 Intrinsic::coro_await_suspend_void,
76 Intrinsic::coro_begin,
77 Intrinsic::coro_begin_custom_abi,
78 Intrinsic::coro_destroy,
79 Intrinsic::coro_done,
80 Intrinsic::coro_end,
81 Intrinsic::coro_end_async,
82 Intrinsic::coro_frame,
83 Intrinsic::coro_free,
84 Intrinsic::coro_id,
85 Intrinsic::coro_id_async,
86 Intrinsic::coro_id_retcon,
87 Intrinsic::coro_id_retcon_once,
88 Intrinsic::coro_noop,
89 Intrinsic::coro_prepare_async,
90 Intrinsic::coro_prepare_retcon,
91 Intrinsic::coro_promise,
92 Intrinsic::coro_resume,
93 Intrinsic::coro_save,
94 Intrinsic::coro_subfn_addr,
95 Intrinsic::coro_suspend,
96 Intrinsic::coro_is_in_ramp,
97};
98
102
106
107
109#ifndef NDEBUG
112 "Only non-overloaded intrinsics supported");
113#endif
114
117 return true;
118 return false;
119}
120
121
122
128
129 if (CoroFrees.empty())
130 return;
131
132 Value *Replacement =
133 Elide
135 : CoroFrees.front()->getFrame();
136
138 CF->replaceAllUsesWith(Replacement);
139 CF->eraseFromParent();
140 }
141}
142
148
149 if (CoroAllocs.empty())
150 return;
151
153}
154
155
156
157
158
159
160
164 for (auto *CA : CoroAllocs) {
165 CA->replaceAllUsesWith(False);
166 CA->eraseFromParent();
167 }
168}
169
178 return SaveInst;
179}
180
181
187
188 bool HasFinalSuspend = false;
189 bool HasUnwindCoroEnd = false;
190 size_t FinalSuspendIndex = 0;
191
193
194
198 switch (II->getIntrinsicID()) {
199 default:
200 continue;
201 case Intrinsic::coro_size:
203 break;
204 case Intrinsic::coro_align:
206 break;
207 case Intrinsic::coro_frame:
209 break;
210 case Intrinsic::coro_save:
211
212
213 if (II->use_empty())
215 break;
216 case Intrinsic::coro_suspend_async: {
218 Suspend->checkWellFormed();
220 break;
221 }
222 case Intrinsic::coro_suspend_retcon: {
225 break;
226 }
227 case Intrinsic::coro_suspend: {
230 if (Suspend->isFinal()) {
231 if (HasFinalSuspend)
233 "Only one suspend point can be marked as final");
234 HasFinalSuspend = true;
236 }
237 break;
238 }
239 case Intrinsic::coro_begin:
240 case Intrinsic::coro_begin_custom_abi: {
242
243
245 if (Id && !Id->getInfo().isPreSplit())
246 break;
247
250 "coroutine should have exactly one defining @llvm.coro.begin");
251 CB->addRetAttr(Attribute::NonNull);
252 CB->addRetAttr(Attribute::NoAlias);
253 CB->removeFnAttr(Attribute::NoDuplicate);
255 break;
256 }
257 case Intrinsic::coro_end_async:
258 case Intrinsic::coro_end:
261 AsyncEnd->checkWellFormed();
262 }
263
264 if (CoroEnds.back()->isUnwind())
265 HasUnwindCoroEnd = true;
266
268
269
270
272 if (CoroEnds.front()->isFallthrough())
274 "Only one coro.end can be marked as fallthrough");
276 }
277 }
278 break;
279 case Intrinsic::coro_is_in_ramp:
281 break;
282 case Intrinsic::coro_promise:
283 assert(CoroPromise == nullptr &&
284 "CoroEarly must ensure coro.promise unique");
286 break;
287 }
288 }
289 }
290
291
293 return;
294
295
297 switch (auto IntrID = Id->getIntrinsicID()) {
298 case Intrinsic::coro_id: {
302
305 SwitchLowering.PromiseAlloca = SwitchId->getPromise();
307
308
310 FinalSuspendIndex != CoroSuspends.size() - 1)
312 break;
313 }
314 case Intrinsic::coro_id_async: {
317 AsyncId->checkWellFormed();
319 AsyncLowering.ContextArgNo = AsyncId->getStorageArgumentIndex();
320 AsyncLowering.ContextHeaderSize = AsyncId->getStorageSize();
321 AsyncLowering.ContextAlignment = AsyncId->getStorageAlignment().value();
322 AsyncLowering.AsyncFuncPointer = AsyncId->getAsyncFunctionPointer();
324 break;
325 }
326 case Intrinsic::coro_id_retcon:
327 case Intrinsic::coro_id_retcon_once: {
331 ContinuationId->checkWellFormed();
332 auto Prototype = ContinuationId->getPrototype();
334 RetconLowering.Alloc = ContinuationId->getAllocFunction();
335 RetconLowering.Dealloc = ContinuationId->getDeallocFunction();
338 break;
339 }
340 default:
341 llvm_unreachable("coro.begin is not dependent on a coro.id call");
342 }
343}
344
345
349 {
350
351
354 CF->replaceAllUsesWith(Poison);
355 CF->eraseFromParent();
356 }
357 CoroFrames.clear();
358
359
360
363 if (auto *CoroSave = CS->getCoroSave())
364 CoroSave->eraseFromParent();
365 CS->eraseFromParent();
366 }
368
369
372 }
373}
374
377 {
378 for (auto *AnySuspend : Shape.CoroSuspends) {
380 if (!Suspend) {
381#ifndef NDEBUG
382 AnySuspend->dump();
383#endif
385 }
386
387 if (!Suspend->getCoroSave())
389 }
390 }
391}
392
394
397 {
398
399
400 auto ResultTys = Shape.getRetconResultTypes();
401 auto ResumeTys = Shape.getRetconResumeTypes();
402
403 for (auto *AnySuspend : Shape.CoroSuspends) {
405 if (!Suspend) {
406#ifndef NDEBUG
407 AnySuspend->dump();
408#endif
410 "coro.suspend.retcon");
411 }
412
413
414 auto SI = Suspend->value_begin(), SE = Suspend->value_end();
415 auto RI = ResultTys.begin(), RE = ResultTys.end();
416 for (; SI != SE && RI != RE; ++SI, ++RI) {
417 auto SrcTy = (*SI)->getType();
418 if (SrcTy != *RI) {
419
420
421
423 auto BCI = new BitCastInst(*SI, *RI, "", Suspend->getIterator());
424 SI->set(BCI);
425 continue;
426 }
427
428#ifndef NDEBUG
429 Suspend->dump();
430 Shape.RetconLowering.ResumePrototype->getFunctionType()->dump();
431#endif
433 "match corresponding prototype function result");
434 }
435 }
436 if (SI != SE || RI != RE) {
437#ifndef NDEBUG
438 Suspend->dump();
439 Shape.RetconLowering.ResumePrototype->getFunctionType()->dump();
440#endif
441 report_fatal_error("wrong number of arguments to coro.suspend.retcon");
442 }
443
444
445 Type *SResultTy = Suspend->getType();
448
450 SuspendResultTys = SResultStructTy->elements();
451 } else {
452
453 SuspendResultTys = SResultTy;
454 }
455 if (SuspendResultTys.size() != ResumeTys.size()) {
456#ifndef NDEBUG
457 Suspend->dump();
458 Shape.RetconLowering.ResumePrototype->getFunctionType()->dump();
459#endif
460 report_fatal_error("wrong number of results from coro.suspend.retcon");
461 }
462 for (size_t I = 0, E = ResumeTys.size(); I != E; ++I) {
463 if (SuspendResultTys[I] != ResumeTys[I]) {
464#ifndef NDEBUG
465 Suspend->dump();
466 Shape.RetconLowering.ResumePrototype->getFunctionType()->dump();
467#endif
469 "match corresponding prototype function param");
470 }
471 }
472 }
473 }
474}
475
479
481 CF->replaceAllUsesWith(CoroBegin);
482 CF->eraseFromParent();
483 }
484 CoroFrames.clear();
485
486
487 for (CoroSaveInst *CoroSave : UnusedCoroSaves)
488 CoroSave->eraseFromParent();
489 UnusedCoroSaves.clear();
490
491 if (PI) {
496 }
497}
498
500 Call->setCallingConv(Callee->getCallingConv());
501
502}
503
505 if (CG)
506 (*CG)[Call->getFunction()]->addCalledFunction(Call, (*CG)[Callee]);
507}
508
511 switch (ABI) {
513 llvm_unreachable("can't allocate memory in coro switch-lowering");
514
518 Size = Builder.CreateIntCast(Size,
519 Alloc->getFunctionType()->getParamType(0),
520 false);
525 }
527 llvm_unreachable("can't allocate memory in coro async-lowering");
528 }
530}
531
534 switch (ABI) {
536 llvm_unreachable("can't allocate memory in coro switch-lowering");
537
541 Ptr = Builder.CreateBitCast(Ptr,
542 Dealloc->getFunctionType()->getParamType(0));
543 auto *Call = Builder.CreateCall(Dealloc, Ptr);
546 return;
547 }
549 llvm_unreachable("can't allocate memory in coro async-lowering");
550 }
552}
553
556#ifndef NDEBUG
557 I->dump();
558 if (V) {
559 errs() << " Value: ";
561 errs() << '\n';
562 }
563#endif
565}
566
567
568
571 if ()
572 fail(I, "llvm.coro.id.retcon.* prototype not a Function", V);
573
574 auto FT = F->getFunctionType();
575
577 bool ResultOkay;
578 if (FT->getReturnType()->isPointerTy()) {
579 ResultOkay = true;
581 ResultOkay = (!SRetTy->isOpaque() &&
582 SRetTy->getNumElements() > 0 &&
583 SRetTy->getElementType(0)->isPointerTy());
584 } else {
585 ResultOkay = false;
586 }
587 if (!ResultOkay)
588 fail(I, "llvm.coro.id.retcon prototype must return pointer as first "
589 "result", F);
590
591 if (FT->getReturnType() !=
592 I->getFunction()->getFunctionType()->getReturnType())
593 fail(I, "llvm.coro.id.retcon prototype return type must be same as"
594 "current function return type", F);
595 } else {
596
597 }
598
599 if (FT->getNumParams() == 0 || !FT->getParamType(0)->isPointerTy())
600 fail(I, "llvm.coro.id.retcon.* prototype must take pointer as "
601 "its first parameter", F);
602}
603
604
607 if ()
608 fail(I, "llvm.coro.* allocator not a Function", V);
609
610 auto FT = F->getFunctionType();
611 if (!FT->getReturnType()->isPointerTy())
612 fail(I, "llvm.coro.* allocator must return a pointer", F);
613
614 if (FT->getNumParams() != 1 ||
615 !FT->getParamType(0)->isIntegerTy())
616 fail(I, "llvm.coro.* allocator must take integer as only param", F);
617}
618
619
622 if ()
623 fail(I, "llvm.coro.* deallocator not a Function", V);
624
625 auto FT = F->getFunctionType();
626 if (!FT->getReturnType()->isVoidTy())
627 fail(I, "llvm.coro.* deallocator must return void", F);
628
629 if (FT->getNumParams() != 1 ||
630 !FT->getParamType(0)->isPointerTy())
631 fail(I, "llvm.coro.* deallocator must take pointer as only param", F);
632}
633
635 const char *Reason) {
638 }
639}
640
643 "size argument to coro.id.retcon.* must be constant");
645 "alignment argument to coro.id.retcon.* must be constant");
649}
650
653 if (!AsyncFuncPtrAddr)
654 fail(I, "llvm.coro.id.async async function pointer not a global", V);
655}
656
659 "size argument to coro.id.async must be constant");
661 "alignment argument to coro.id.async must be constant");
663 "storage argument offset to coro.id.async must be constant");
665}
666
670 if (!FunTy->getReturnType()->isPointerTy())
672 "llvm.coro.suspend.async resume function projection function must "
673 "return a ptr type",
674 F);
675 if (FunTy->getNumParams() != 1 || !FunTy->getParamType(0)->isPointerTy())
677 "llvm.coro.suspend.async resume function projection function must "
678 "take one ptr type as parameter",
679 F);
680}
681
685
688 if (!MustTailCallFunc)
689 return;
690 auto *FnTy = MustTailCallFunc->getFunctionType();
691 if (FnTy->getNumParams() != (arg_size() - 3))
693 "llvm.coro.end.async must tail call function argument type must "
694 "match the tail arguments",
695 MustTailCallFunc);
696}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg, SDValue Val={})
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static Intrinsic::ID NonOverloadedCoroIntrinsics[]
Definition Coroutines.cpp:67
static void checkWFDealloc(const Instruction *I, Value *V)
Check that the given value is a well-formed deallocator.
Definition Coroutines.cpp:620
static void checkConstantInt(const Instruction *I, Value *V, const char *Reason)
Definition Coroutines.cpp:634
static void checkWFRetconPrototype(const AnyCoroIdRetconInst *I, Value *V)
Check that the given value is a well-formed prototype for the llvm.coro.id.retcon.
Definition Coroutines.cpp:569
static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee)
Definition Coroutines.cpp:499
static void checkAsyncContextProjectFunction(const Instruction *I, Function *F)
Definition Coroutines.cpp:667
static CoroSaveInst * createCoroSave(CoroBeginInst *CoroBegin, CoroSuspendInst *SuspendInst)
Definition Coroutines.cpp:170
static void checkWFAlloc(const Instruction *I, Value *V)
Check that the given value is a well-formed allocator.
Definition Coroutines.cpp:605
static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee)
Definition Coroutines.cpp:504
static void checkAsyncFuncPointer(const Instruction *I, Value *V)
Definition Coroutines.cpp:651
Module.h This file contains the declarations for the Module class.
uint64_t IntrinsicInst * II
This file defines the SmallVector class.
This represents either the llvm.coro.id.retcon or llvm.coro.id.retcon.once instruction.
LLVM_ABI void checkWellFormed() const
Definition Coroutines.cpp:641
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
LLVM Basic Block Representation.
const Instruction & front() const
This class represents a no-op cast from one type to another.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
unsigned arg_size() const
The basic data container for the call graph of a Module of IR.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
static LLVM_ABI bool isBitCastable(Type *SrcTy, Type *DestTy)
Check whether a bitcast between these types is valid.
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
A constant pointer value that points to null.
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Function * getMustTailCallFunction() const
LLVM_ABI void checkWellFormed() const
Definition Coroutines.cpp:686
This class represents the llvm.coro.begin or llvm.coro.begin.custom.abi instructions.
This represents the llvm.coro.frame instruction.
This represents the llvm.coro.free instruction.
LLVM_ABI void checkWellFormed() const
Definition Coroutines.cpp:657
This represents the llvm.coro.id instruction.
This represents the llvm.coro.promise instruction.
bool isFromPromise() const
Are we translating from the frame to the promise (false) or from the promise to the frame (true)?
This represents the llvm.coro.save instruction.
Function * getAsyncContextProjectionFunction() const
LLVM_ABI void checkWellFormed() const
Definition Coroutines.cpp:682
This represents the llvm.coro.suspend instruction.
CoroSaveInst * getCoroSave() const
Class to represent function types.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
This is an important class for using LLVM in a threaded context.
A Module instance is used to store all the information related to an LLVM module.
Class to represent pointers.
static LLVM_ABI PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
LLVM Value Representation.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
void init() override
Definition Coroutines.cpp:395
void init() override
Definition Coroutines.cpp:393
void init() override
Definition Coroutines.cpp:375
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
LLVM_ABI Function * getDeclarationIfExists(const Module *M, ID id)
Look up the Function declaration of the intrinsic id in the Module M and return it if it exists.
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
@ Async
The "async continuation" lowering, where each suspend point creates a single continuation function.
@ RetconOnce
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
@ Retcon
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
bool declaresAnyIntrinsic(const Module &M)
Definition Coroutines.cpp:103
bool isSuspendBlock(BasicBlock *BB)
Definition Coroutines.cpp:99
void suppressCoroAllocs(CoroIdInst *CoroId)
Replaces all @llvm.coro.alloc intrinsics calls associated with a given call @llvm....
Definition Coroutines.cpp:143
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition Coroutines.cpp:123
bool declaresIntrinsics(const Module &M, ArrayRef< Intrinsic::ID > List)
Definition Coroutines.cpp:108
This is an optimization pass for GlobalISel generic memory operations.
decltype(auto) dyn_cast(const From &Val)
dyn_cast - Return the argument parameter cast to the specified type.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
bool isa(const From &Val)
isa - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
decltype(auto) cast(const From &Val)
cast - Return the argument parameter cast to the specified type.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
PointerType *const Int8Ptr
LowererBase(Module &M)
Definition Coroutines.cpp:40
ConstantPointerNull *const NullPtr
CallInst * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
Definition Coroutines.cpp:52
FunctionType *const ResumeFnType
SmallVector< CoroAwaitSuspendInst *, 4 > CoroAwaitSuspends
AsyncLoweringStorage AsyncLowering
LLVM_ABI void cleanCoroutine(SmallVectorImpl< CoroFrameInst * > &CoroFrames, SmallVectorImpl< CoroSaveInst * > &UnusedCoroSaves, CoroPromiseInst *CoroPromise)
Definition Coroutines.cpp:476
AnyCoroIdRetconInst * getRetconCoroId() const
CoroIdInst * getSwitchCoroId() const
SmallVector< CoroSizeInst *, 2 > CoroSizes
LLVM_ABI void analyze(Function &F, SmallVectorImpl< CoroFrameInst * > &CoroFrames, SmallVectorImpl< CoroSaveInst * > &UnusedCoroSaves, CoroPromiseInst *&CoroPromise)
Definition Coroutines.cpp:182
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
LLVM_ABI Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition Coroutines.cpp:509
AllocaInst * getPromiseAlloca() const
SwitchLoweringStorage SwitchLowering
CoroBeginInst * CoroBegin
SmallVector< CoroIsInRampInst *, 2 > CoroIsInRampInsts
LLVM_ABI void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition Coroutines.cpp:532
RetconLoweringStorage RetconLowering
SmallVector< CoroAlignInst *, 2 > CoroAligns
CoroIdAsyncInst * getAsyncCoroId() const
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
LLVM_ABI void invalidateCoroutine(Function &F, SmallVectorImpl< CoroFrameInst * > &CoroFrames)
Definition Coroutines.cpp:346