CoroSplit.cpp File Reference (original) (raw)

Go to the source code of this file.

Macros
#define	DEBUG_TYPE "coro-split"

Functions
static void	lowerAwaitSuspend (IRBuilder<> &Builder, CoroAwaitSuspendInst *CB, coro::Shape &Shape)
static void	lowerAwaitSuspends (Function &F, coro::Shape &Shape)
static void	maybeFreeRetconStorage (IRBuilder<> &Builder, const coro::Shape &Shape, Value FramePtr, CallGraph CG)
static bool	replaceCoroEndAsync (AnyCoroEndInst *End)
	Replace an llvm.coro.end.async.
static void	replaceFallthroughCoroEnd (AnyCoroEndInst End, const coro::Shape &Shape, Value FramePtr, bool InRamp, CallGraph *CG)
	Replace a non-unwind call to llvm.coro.end.
static void	markCoroutineAsDone (IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr)
static void	replaceUnwindCoroEnd (AnyCoroEndInst End, const coro::Shape &Shape, Value FramePtr, bool InRamp, CallGraph *CG)
	Replace an unwind call to llvm.coro.end.
static void	replaceCoroEnd (AnyCoroEndInst End, const coro::Shape &Shape, Value FramePtr, bool InRamp, CallGraph *CG)
static FunctionType *	getFunctionTypeFromAsyncSuspend (AnyCoroSuspendInst *Suspend)
static Function *	createCloneDeclaration (Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)
static void	replaceSwiftErrorOps (Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)
static SmallVector< DbgVariableRecord * >	collectDbgVariableRecords (Function &F)
	Returns all debug records in F.
static void	updateScopeLine (Instruction *ActiveSuspend, DISubprogram &SPToUpdate)
	Adjust the scope line of the funclet to the first line number after the suspend point.
static void	addFramePointerAttrs (AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment, bool NoAlias)
static void	addAsyncContextAttrs (AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
static void	addSwiftSelfAttrs (AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
static void	updateAsyncFuncPointerContextSize (coro::Shape &Shape)
static TypeSize	getFrameSizeForShape (coro::Shape &Shape)
static void	replaceFrameSizeAndAlignment (coro::Shape &Shape)
static void	postSplitCleanup (Function &F)
static void	handleNoSuspendCoroutine (coro::Shape &Shape)
static bool	hasCallsInBlockBetween (iterator_range< BasicBlock::iterator > R)
static bool	hasCallsInBlocksBetween (BasicBlock SaveBB, BasicBlock ResDesBB)
static bool	hasCallsBetween (Instruction Save, Instruction ResumeOrDestroy)
static bool	simplifySuspendPoint (CoroSuspendInst Suspend, CoroBeginInst CoroBegin)
static void	simplifySuspendPoints (coro::Shape &Shape)
static void	replaceAsyncResumeFunction (CoroSuspendAsyncInst Suspend, Value Continuation)
static void	coerceArguments (IRBuilder<> &Builder, FunctionType FnTy, ArrayRef< Value > FnArgs, SmallVectorImpl< Value * > &CallArgs)
	Coerce the arguments in FnArgs according to FnTy in CallArgs.
static void	removeCoroEndsFromRampFunction (const coro::Shape &Shape)
	Remove calls to llvm.coro.end in the original function.
static void	removeCoroIsInRampFromRampFunction (const coro::Shape &Shape)
static bool	hasSafeElideCaller (Function &F)
static void	doSplitCoroutine (Function &F, SmallVectorImpl< Function * > &Clones, coro::BaseABI &ABI, TargetTransformInfo &TTI, bool OptimizeFrame)
static LazyCallGraph::SCC &	updateCallGraphAfterCoroutineSplit (LazyCallGraph::Node &N, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, LazyCallGraph::SCC &C, LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
static void	replacePrepare (CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)
	Replace a call to llvm.coro.prepare.retcon.
static bool	replaceAllPrepares (Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)
static void	addPrepareFunction (const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)
static std::unique_ptr< coro::BaseABI >	CreateNewABI (Function &F, coro::Shape &S, std::function< bool(Instruction &)> IsMatCallback, const SmallVector< CoroSplitPass::BaseABITy > GenCustomABIs)

◆ DEBUG_TYPE

#define DEBUG_TYPE "coro-split"

◆ addAsyncContextAttrs()

void addAsyncContextAttrs ( AttributeList & Attrs, LLVMContext & Context, unsigned ParamIndex )	static

◆ addFramePointerAttrs()

◆ addPrepareFunction()

◆ addSwiftSelfAttrs()

void addSwiftSelfAttrs ( AttributeList & Attrs, LLVMContext & Context, unsigned ParamIndex )	static

◆ coerceArguments()

◆ collectDbgVariableRecords()

◆ createCloneDeclaration()

◆ CreateNewABI()

Definition at line 2145 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::coro::Shape::CoroBegin, F, llvm::CoroBeginInst::getCustomABI(), llvm::CoroBeginInst::hasCustomABI(), llvm_unreachable, llvm::coro::Retcon, llvm::coro::RetconOnce, llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::coro::Switch.

Referenced by llvm::CoroSplitPass::CoroSplitPass(), llvm::CoroSplitPass::CoroSplitPass(), llvm::CoroSplitPass::CoroSplitPass(), and llvm::CoroSplitPass::CoroSplitPass().

◆ doSplitCoroutine()

Definition at line 1999 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, assert(), collectDbgVariableRecords(), llvm::coro::Shape::CoroBegin, llvm::coro::Shape::CoroSuspends, F, handleNoSuspendCoroutine(), hasSafeElideCaller(), lowerAwaitSuspends(), llvm::coro::normalizeCoroutine(), removeCoroEndsFromRampFunction(), removeCoroIsInRampFromRampFunction(), replaceFrameSizeAndAlignment(), replaceSwiftErrorOps(), llvm::coro::salvageDebugInfo(), llvm::coro::Shape::Shape(), simplifySuspendPoints(), and llvm::coro::Switch.

Referenced by llvm::CoroSplitPass::run().

◆ getFrameSizeForShape()

◆ getFunctionTypeFromAsyncSuspend()

◆ handleNoSuspendCoroutine()

void handleNoSuspendCoroutine ( coro::Shape & Shape)	static

Definition at line 1165 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::coro::Shape::CoroBegin, llvm::Instruction::eraseFromParent(), llvm::coro::Shape::FrameAlign, llvm::coro::Shape::FrameTy, llvm::PoisonValue::get(), llvm::AnyCoroIdInst::getCoroAlloc(), llvm::coro::Shape::getSwitchCoroId(), llvm::coro::replaceCoroFree(), llvm::coro::Retcon, llvm::coro::RetconOnce, and llvm::coro::Switch.

Referenced by doSplitCoroutine().

◆ hasCallsBetween()

◆ hasCallsInBlockBetween()

◆ hasCallsInBlocksBetween()

◆ hasSafeElideCaller()

◆ lowerAwaitSuspend()

Definition at line 86 of file CoroSplit.cpp.

References Call, llvm::dyn_cast(), llvm::Instruction::eraseFromParent(), llvm::CallingConv::Fast, FramePtr, llvm::FunctionType::get(), llvm::CallBase::getAttributes(), llvm::CoroAwaitSuspendInst::getAwaiter(), llvm::CallBase::getCalledFunction(), llvm::Value::getContext(), llvm::CoroAwaitSuspendInst::getFrame(), llvm::Function::getIntrinsicID(), llvm::PointerType::getUnqual(), llvm::Type::getVoidTy(), llvm::CoroAwaitSuspendInst::getWrapperFunction(), llvm_unreachable, llvm::coro::LowererBase::makeSubFnCall(), llvm::Value::replaceAllUsesWith(), llvm::CoroSubFnInst::ResumeIndex, llvm::coro::Shape::SymmetricTransfers, and Wrapper.

Referenced by lowerAwaitSuspends().

◆ lowerAwaitSuspends()

◆ markCoroutineAsDone()

Definition at line 313 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, assert(), llvm::cast(), llvm::coro::Shape::CoroSuspends, FramePtr, llvm::coro::Shape::FrameTy, llvm::ConstantPointerNull::get(), llvm::coro::Shape::getIndex(), llvm::coro::Shape::getSwitchIndexField(), llvm::StructType::getTypeAtIndex(), llvm::coro::Shape::SwitchLoweringStorage::HasFinalSuspend, llvm::coro::Shape::SwitchLoweringStorage::HasUnwindCoroEnd, llvm::coro::Shape::SwitchFieldIndex::Resume, llvm::coro::Switch, and llvm::coro::Shape::SwitchLowering.

Referenced by replaceUnwindCoroEnd().

◆ maybeFreeRetconStorage()

◆ postSplitCleanup()

void postSplitCleanup ( Function & F)	static

◆ removeCoroEndsFromRampFunction()

◆ removeCoroIsInRampFromRampFunction()

◆ replaceAllPrepares()

◆ replaceAsyncResumeFunction()

◆ replaceCoroEnd()

◆ replaceCoroEndAsync()

◆ replaceFallthroughCoroEnd()

Replace a non-unwind call to llvm.coro.end.

Definition at line 214 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, assert(), llvm::coro::Async, llvm::cast(), llvm::dyn_cast(), FramePtr, llvm::ConstantPointerNull::get(), llvm::ConstantTokenNone::get(), llvm::PoisonValue::get(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::coro::Shape::getResumeFunctionType(), llvm::FunctionType::getReturnType(), maybeFreeRetconStorage(), replaceCoroEndAsync(), llvm::coro::Retcon, llvm::coro::RetconOnce, and llvm::coro::Switch.

Referenced by replaceCoroEnd().

◆ replaceFrameSizeAndAlignment()

void replaceFrameSizeAndAlignment ( coro::Shape & Shape)	static

Definition at line 1127 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::coro::Shape::CoroAligns, llvm::coro::Shape::CoroSizes, llvm::Instruction::eraseFromParent(), llvm::coro::Shape::FrameAlign, getFrameSizeForShape(), llvm::Value::getType(), llvm::Value::replaceAllUsesWith(), updateAsyncFuncPointerContextSize(), and llvm::Align::value().

Referenced by doSplitCoroutine().

◆ replacePrepare()

◆ replaceSwiftErrorOps()

Definition at line 569 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, assert(), llvm::coro::Async, llvm::cast(), llvm::coro::Shape::CoroSuspends, F, llvm::Value::getType(), llvm::Value::replaceAllUsesWith(), and llvm::coro::Shape::SwiftErrorOps.

Referenced by llvm::coro::BaseCloner::create(), doSplitCoroutine(), and llvm::coro::BaseCloner::replaceSwiftErrorOps().

◆ replaceUnwindCoroEnd()

Replace an unwind call to llvm.coro.end.

Definition at line 347 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::cast(), FramePtr, llvm::CallBase::getOperandBundle(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), markCoroutineAsDone(), maybeFreeRetconStorage(), llvm::LLVMContext::OB_funclet, llvm::coro::Retcon, llvm::coro::RetconOnce, and llvm::coro::Switch.

Referenced by replaceCoroEnd().

◆ simplifySuspendPoint()

Definition at line 1267 of file CoroSplit.cpp.

References llvm::BranchInst::Create(), llvm::dyn_cast(), llvm::Instruction::eraseFromParent(), llvm::CallBase::getCalledOperand(), llvm::CoroSuspendInst::getCoroSave(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::ilist_node_with_parent< NodeTy, ParentTy, Options >::getPrevNode(), hasCallsBetween(), I, llvm::Value::replaceAllUsesWith(), and llvm::Value::stripPointerCasts().

Referenced by simplifySuspendPoints().

◆ simplifySuspendPoints()

void simplifySuspendPoints ( coro::Shape & Shape)	static

Definition at line 1327 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, assert(), llvm::cast(), llvm::coro::Shape::CoroBegin, llvm::coro::Shape::CoroSuspends, llvm::coro::Shape::SwitchLoweringStorage::HasFinalSuspend, I, N, simplifySuspendPoint(), std::swap(), llvm::coro::Switch, and llvm::coro::Shape::SwitchLowering.

Referenced by doSplitCoroutine().

◆ updateAsyncFuncPointerContextSize()

void updateAsyncFuncPointerContextSize ( coro::Shape & Shape)	static

◆ updateCallGraphAfterCoroutineSplit()

Definition at line 2048 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, llvm::LazyCallGraph::addSplitFunction(), llvm::LazyCallGraph::addSplitRefRecursiveFunctions(), llvm::coro::Async, llvm::CallingConv::C, llvm::SmallVectorTemplateCommon< T, typename >::empty(), FAM, N, postSplitCleanup(), llvm::coro::Retcon, llvm::coro::RetconOnce, llvm::coro::Switch, llvm::updateCGAndAnalysisManagerForCGSCCPass(), and llvm::updateCGAndAnalysisManagerForFunctionPass().

Referenced by llvm::CoroSplitPass::run().

◆ updateScopeLine()

Adjust the scope line of the funclet to the first line number after the suspend point.

This avoids a jump in the line table from the function declaration (where prologue instructions are attributed to) to the suspend point. Only adjust the scope line when the files are the same. If no candidate line number is found, fallback to the line of ActiveSuspend.

Definition at line 802 of file CoroSplit.cpp.

References DL, llvm::dyn_cast_or_null(), llvm::BasicBlock::end(), llvm::Instruction::getDebugLoc(), llvm::DIScope::getFile(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::ilist_node_with_parent< NodeTy, ParentTy, Options >::getNextNode(), llvm::skipDebugIntrinsics(), and llvm::Successor.

Referenced by llvm::coro::BaseCloner::create().