Coroutines.cpp Source File (original) (raw)

33#include

34#include

35#include

37using namespace llvm;

41 : TheModule(M), Context(M.getContext()),

44 false)),

54 auto *IndexVal = ConstantInt::get(Type::getInt8Ty(Context), Index);

55 auto *Fn =

60 "makeSubFnCall: Index value out of range");

62}

66 "llvm.coro.align",

67 "llvm.coro.alloc",

68 "llvm.coro.async.context.alloc",

69 "llvm.coro.async.context.dealloc",

70 "llvm.coro.async.resume",

71 "llvm.coro.async.size.replace",

72 "llvm.coro.await.suspend.bool",

73 "llvm.coro.await.suspend.handle",

74 "llvm.coro.await.suspend.void",

75 "llvm.coro.begin",

76 "llvm.coro.begin.custom.abi",

77 "llvm.coro.destroy",

78 "llvm.coro.done",

79 "llvm.coro.end",

80 "llvm.coro.end.async",

81 "llvm.coro.frame",

82 "llvm.coro.free",

83 "llvm.coro.id",

84 "llvm.coro.id.async",

85 "llvm.coro.id.retcon",

86 "llvm.coro.id.retcon.once",

87 "llvm.coro.noop",

88 "llvm.coro.prepare.async",

89 "llvm.coro.prepare.retcon",

90 "llvm.coro.promise",

91 "llvm.coro.resume",

92 "llvm.coro.save",

93 "llvm.coro.size",

94 "llvm.coro.subfn.addr",

95 "llvm.coro.suspend",

96 "llvm.coro.suspend.async",

97 "llvm.coro.suspend.retcon",

98};

100#ifndef NDEBUG

103}

104#endif

105

107 return isa(BB->front());

108}

109

112 if (M.getNamedValue(Name))

113 return true;

114 }

115

116 return false;

117}

118

119

120

122 const std::initializer_list List) {

125 if (M.getNamedValue(Name))

126 return true;

127 }

128

129 return false;

130}

131

132

133

137 if (auto CF = dyn_cast(U))

139

140 if (CoroFrees.empty())

141 return;

142

143 Value *Replacement =

144 Elide

146 : CoroFrees.front()->getFrame();

147

149 CF->replaceAllUsesWith(Replacement);

150 CF->eraseFromParent();

151 }

152}

153

157 if (auto *CA = dyn_cast(U))

159

160 if (CoroAllocs.empty())

161 return;

162

164}

165

166

167

168

169

170

171

175 for (auto *CA : CoroAllocs) {

176 CA->replaceAllUsesWith(False);

177 CA->eraseFromParent();

178 }

179}

180

185 auto *SaveInst = cast(

189 return SaveInst;

190}

191

192

196 clear();

197

198 bool HasFinalSuspend = false;

199 bool HasUnwindCoroEnd = false;

200 size_t FinalSuspendIndex = 0;

201

203

204

205 if (auto AWS = dyn_cast(&I)) {

206 CoroAwaitSuspends.push_back(AWS);

207 } else if (auto II = dyn_cast(&I)) {

208 switch (II->getIntrinsicID()) {

209 default:

210 continue;

211 case Intrinsic::coro_size:

212 CoroSizes.push_back(cast(II));

213 break;

214 case Intrinsic::coro_align:

215 CoroAligns.push_back(cast(II));

216 break;

217 case Intrinsic::coro_frame:

218 CoroFrames.push_back(cast(II));

219 break;

220 case Intrinsic::coro_save:

221

222

223 if (II->use_empty())

224 UnusedCoroSaves.push_back(cast(II));

225 break;

226 case Intrinsic::coro_suspend_async: {

227 auto *Suspend = cast(II);

228 Suspend->checkWellFormed();

229 CoroSuspends.push_back(Suspend);

230 break;

231 }

232 case Intrinsic::coro_suspend_retcon: {

233 auto Suspend = cast(II);

234 CoroSuspends.push_back(Suspend);

235 break;

236 }

237 case Intrinsic::coro_suspend: {

238 auto Suspend = cast(II);

239 CoroSuspends.push_back(Suspend);

240 if (Suspend->isFinal()) {

241 if (HasFinalSuspend)

243 "Only one suspend point can be marked as final");

244 HasFinalSuspend = true;

245 FinalSuspendIndex = CoroSuspends.size() - 1;

246 }

247 break;

248 }

249 case Intrinsic::coro_begin:

250 case Intrinsic::coro_begin_custom_abi: {

251 auto CB = cast(II);

252

253

254 auto Id = dyn_cast(CB->getId());

255 if (Id && !Id->getInfo().isPreSplit())

256 break;

257

258 if (CoroBegin)

260 "coroutine should have exactly one defining @llvm.coro.begin");

261 CB->addRetAttr(Attribute::NonNull);

262 CB->addRetAttr(Attribute::NoAlias);

263 CB->removeFnAttr(Attribute::NoDuplicate);

264 CoroBegin = CB;

265 break;

266 }

267 case Intrinsic::coro_end_async:

268 case Intrinsic::coro_end:

269 CoroEnds.push_back(cast(II));

270 if (auto *AsyncEnd = dyn_cast(II)) {

271 AsyncEnd->checkWellFormed();

272 }

273

274 if (CoroEnds.back()->isUnwind())

275 HasUnwindCoroEnd = true;

276

277 if (CoroEnds.back()->isFallthrough() && isa(II)) {

278

279

280

281 if (CoroEnds.size() > 1) {

282 if (CoroEnds.front()->isFallthrough())

284 "Only one coro.end can be marked as fallthrough");

285 std::swap(CoroEnds.front(), CoroEnds.back());

286 }

287 }

288 break;

289 }

290 }

291 }

292

293

294 if (!CoroBegin)

295 return;

296

297

298 auto Id = CoroBegin->getId();

299 switch (auto IntrID = Id->getIntrinsicID()) {

300 case Intrinsic::coro_id: {

302 SwitchLowering.HasFinalSuspend = HasFinalSuspend;

303 SwitchLowering.HasUnwindCoroEnd = HasUnwindCoroEnd;

304

305 auto SwitchId = getSwitchCoroId();

306 SwitchLowering.ResumeSwitch = nullptr;

307 SwitchLowering.PromiseAlloca = SwitchId->getPromise();

308 SwitchLowering.ResumeEntryBlock = nullptr;

309

310

311 if (SwitchLowering.HasFinalSuspend &&

312 FinalSuspendIndex != CoroSuspends.size() - 1)

313 std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back());

314 break;

315 }

316 case Intrinsic::coro_id_async: {

318 auto *AsyncId = getAsyncCoroId();

319 AsyncId->checkWellFormed();

320 AsyncLowering.Context = AsyncId->getStorage();

321 AsyncLowering.ContextArgNo = AsyncId->getStorageArgumentIndex();

322 AsyncLowering.ContextHeaderSize = AsyncId->getStorageSize();

323 AsyncLowering.ContextAlignment = AsyncId->getStorageAlignment().value();

324 AsyncLowering.AsyncFuncPointer = AsyncId->getAsyncFunctionPointer();

325 AsyncLowering.AsyncCC = F.getCallingConv();

326 break;

327 }

328 case Intrinsic::coro_id_retcon:

329 case Intrinsic::coro_id_retcon_once: {

332 auto ContinuationId = getRetconCoroId();

333 ContinuationId->checkWellFormed();

334 auto Prototype = ContinuationId->getPrototype();

335 RetconLowering.ResumePrototype = Prototype;

336 RetconLowering.Alloc = ContinuationId->getAllocFunction();

337 RetconLowering.Dealloc = ContinuationId->getDeallocFunction();

338 RetconLowering.ReturnBlock = nullptr;

339 RetconLowering.IsFrameInlineInStorage = false;

340 break;

341 }

342 default:

343 llvm_unreachable("coro.begin is not dependent on a coro.id call");

344 }

345}

346

347

351 {

352

353

356 CF->replaceAllUsesWith(Poison);

357 CF->eraseFromParent();

358 }

359 CoroFrames.clear();

360

361

362

365 CS->eraseFromParent();

366 if (auto *CoroSave = CS->getCoroSave())

367 CoroSave->eraseFromParent();

368 }

369 CoroSuspends.clear();

370

371

374 }

375}

376

379 {

381 auto Suspend = dyn_cast(AnySuspend);

382 if (!Suspend) {

383#ifndef NDEBUG

384 AnySuspend->dump();

385#endif

387 }

388

389 if (!Suspend->getCoroSave())

391 }

392 }

393}

394

396

399 {

400

401

404

406 auto Suspend = dyn_cast(AnySuspend);

407 if (!Suspend) {

408#ifndef NDEBUG

409 AnySuspend->dump();

410#endif

412 "coro.suspend.retcon");

413 }

414

415

416 auto SI = Suspend->value_begin(), SE = Suspend->value_end();

417 auto RI = ResultTys.begin(), RE = ResultTys.end();

418 for (; SI != SE && RI != RE; ++SI, ++RI) {

419 auto SrcTy = (*SI)->getType();

420 if (SrcTy != *RI) {

421

422

423

425 auto BCI = new BitCastInst(*SI, *RI, "", Suspend->getIterator());

426 SI->set(BCI);

427 continue;

428 }

429

430#ifndef NDEBUG

431 Suspend->dump();

433#endif

435 "match corresponding prototype function result");

436 }

437 }

438 if (SI != SE || RI != RE) {

439#ifndef NDEBUG

440 Suspend->dump();

442#endif

443 report_fatal_error("wrong number of arguments to coro.suspend.retcon");

444 }

445

446

447 Type *SResultTy = Suspend->getType();

450

451 } else if (auto SResultStructTy = dyn_cast(SResultTy)) {

452 SuspendResultTys = SResultStructTy->elements();

453 } else {

454

455 SuspendResultTys = SResultTy;

456 }

457 if (SuspendResultTys.size() != ResumeTys.size()) {

458#ifndef NDEBUG

459 Suspend->dump();

461#endif

462 report_fatal_error("wrong number of results from coro.suspend.retcon");

463 }

464 for (size_t I = 0, E = ResumeTys.size(); I != E; ++I) {

465 if (SuspendResultTys[I] != ResumeTys[I]) {

466#ifndef NDEBUG

467 Suspend->dump();

469#endif

471 "match corresponding prototype function param");

472 }

473 }

474 }

475 }

476}

477

481

483 CF->replaceAllUsesWith(CoroBegin);

484 CF->eraseFromParent();

485 }

486 CoroFrames.clear();

487

488

489 for (CoroSaveInst *CoroSave : UnusedCoroSaves)

490 CoroSave->eraseFromParent();

491 UnusedCoroSaves.clear();

492}

493

495 Call->setCallingConv(Callee->getCallingConv());

496

497}

498

500 if (CG)

501 (*CG)[Call->getFunction()]->addCalledFunction(Call, (*CG)[Callee]);

502}

503

506 switch (ABI) {

508 llvm_unreachable("can't allocate memory in coro switch-lowering");

509

512 auto Alloc = RetconLowering.Alloc;

514 Alloc->getFunctionType()->getParamType(0),

515 false);

519 return Call;

520 }

522 llvm_unreachable("can't allocate memory in coro async-lowering");

523 }

525}

526

529 switch (ABI) {

531 llvm_unreachable("can't allocate memory in coro switch-lowering");

532

535 auto Dealloc = RetconLowering.Dealloc;

537 Dealloc->getFunctionType()->getParamType(0));

541 return;

542 }

544 llvm_unreachable("can't allocate memory in coro async-lowering");

545 }

547}

548

551#ifndef NDEBUG

552 I->dump();

553 if (V) {

554 errs() << " Value: ";

556 errs() << '\n';

557 }

558#endif

560}

561

562

563

565 auto F = dyn_cast(V->stripPointerCasts());

566 if ()

567 fail(I, "llvm.coro.id.retcon.* prototype not a Function", V);

568

569 auto FT = F->getFunctionType();

570

571 if (isa(I)) {

572 bool ResultOkay;

573 if (FT->getReturnType()->isPointerTy()) {

574 ResultOkay = true;

575 } else if (auto SRetTy = dyn_cast(FT->getReturnType())) {

576 ResultOkay = (!SRetTy->isOpaque() &&

577 SRetTy->getNumElements() > 0 &&

578 SRetTy->getElementType(0)->isPointerTy());

579 } else {

580 ResultOkay = false;

581 }

582 if (!ResultOkay)

583 fail(I, "llvm.coro.id.retcon prototype must return pointer as first "

584 "result", F);

585

586 if (FT->getReturnType() !=

587 I->getFunction()->getFunctionType()->getReturnType())

588 fail(I, "llvm.coro.id.retcon prototype return type must be same as"

589 "current function return type", F);

590 } else {

591

592 }

593

594 if (FT->getNumParams() == 0 || !FT->getParamType(0)->isPointerTy())

595 fail(I, "llvm.coro.id.retcon.* prototype must take pointer as "

596 "its first parameter", F);

597}

598

599

601 auto F = dyn_cast(V->stripPointerCasts());

602 if ()

603 fail(I, "llvm.coro.* allocator not a Function", V);

604

605 auto FT = F->getFunctionType();

606 if (!FT->getReturnType()->isPointerTy())

607 fail(I, "llvm.coro.* allocator must return a pointer", F);

608

609 if (FT->getNumParams() != 1 ||

610 !FT->getParamType(0)->isIntegerTy())

611 fail(I, "llvm.coro.* allocator must take integer as only param", F);

612}

613

614

616 auto F = dyn_cast(V->stripPointerCasts());

617 if ()

618 fail(I, "llvm.coro.* deallocator not a Function", V);

619

620 auto FT = F->getFunctionType();

621 if (!FT->getReturnType()->isVoidTy())

622 fail(I, "llvm.coro.* deallocator must return void", F);

623

624 if (FT->getNumParams() != 1 ||

625 !FT->getParamType(0)->isPointerTy())

626 fail(I, "llvm.coro.* deallocator must take pointer as only param", F);

627}

628

630 const char *Reason) {

631 if (!isa(V)) {

632 fail(I, Reason, V);

633 }

634}

635

638 "size argument to coro.id.retcon.* must be constant");

640 "alignment argument to coro.id.retcon.* must be constant");

644}

645

647 auto *AsyncFuncPtrAddr = dyn_cast(V->stripPointerCasts());

648 if (!AsyncFuncPtrAddr)

649 fail(I, "llvm.coro.id.async async function pointer not a global", V);

650}

651

654 "size argument to coro.id.async must be constant");

656 "alignment argument to coro.id.async must be constant");

658 "storage argument offset to coro.id.async must be constant");

660}

661

664 auto *FunTy = cast(F->getValueType());

665 if (!FunTy->getReturnType()->isPointerTy())

667 "llvm.coro.suspend.async resume function projection function must "

668 "return a ptr type",

669 F);

670 if (FunTy->getNumParams() != 1 || !FunTy->getParamType(0)->isPointerTy())

672 "llvm.coro.suspend.async resume function projection function must "

673 "take one ptr type as parameter",

674 F);

675}

676

679}

680

682 auto *MustTailCallFunc = getMustTailCallFunction();

683 if (!MustTailCallFunc)

684 return;

685 auto *FnTy = MustTailCallFunc->getFunctionType();

686 if (FnTy->getNumParams() != (arg_size() - 3))

688 "llvm.coro.end.async must tail call function argument type must "

689 "match the tail arguments",

690 MustTailCallFunc);

691}

Expand Atomic instructions

This file contains the simple types necessary to represent the attributes associated with functions a...

static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg, SDValue Val={})

This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...

This file contains the declarations for the subclasses of Constant, which represent the different fla...

static void checkWFDealloc(const Instruction *I, Value *V)

Check that the given value is a well-formed deallocator.

static bool isCoroutineIntrinsicName(StringRef Name)

static void checkConstantInt(const Instruction *I, Value *V, const char *Reason)

static void checkWFRetconPrototype(const AnyCoroIdRetconInst *I, Value *V)

Check that the given value is a well-formed prototype for the llvm.coro.id.retcon.

static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee)

static void checkAsyncContextProjectFunction(const Instruction *I, Function *F)

static const char *const CoroIntrinsics[]

static CoroSaveInst * createCoroSave(CoroBeginInst *CoroBegin, CoroSuspendInst *SuspendInst)

static void checkWFAlloc(const Instruction *I, Value *V)

Check that the given value is a well-formed allocator.

static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee)

static void checkAsyncFuncPointer(const Instruction *I, Value *V)

Module.h This file contains the declarations for the Module class.

uint64_t IntrinsicInst * II

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This file defines the SmallVector class.

This represents either the llvm.coro.id.retcon or llvm.coro.id.retcon.once instruction.

void checkWellFormed() const

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

LLVM Basic Block Representation.

const Instruction & front() const

This class represents a no-op cast from one type to another.

void setArgOperand(unsigned i, Value *v)

The basic data container for the call graph of a Module of IR.

This class represents a function call, abstracting a target machine's calling convention.

static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

static bool isBitCastable(Type *SrcTy, Type *DestTy)

Check whether a bitcast between these types is valid.

static ConstantInt * getFalse(LLVMContext &Context)

A constant pointer value that points to null.

static ConstantPointerNull * get(PointerType *T)

Static factory methods - Return objects of the specified value.

void checkWellFormed() const

This class represents the llvm.coro.begin or llvm.coro.begin.custom.abi instructions.

This represents the llvm.coro.frame instruction.

This represents the llvm.coro.free instruction.

void checkWellFormed() const

This represents the llvm.coro.id instruction.

This represents the llvm.coro.save instruction.

void checkWellFormed() const

This represents the llvm.coro.suspend instruction.

CoroSaveInst * getCoroSave() const

Class to represent function types.

FunctionType * getFunctionType() const

Returns the FunctionType for me.

Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")

CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

const Module * getModule() const

Return the module owning the function this instruction belongs to or nullptr it the function does not...

This is an important class for using LLVM in a threaded context.

A Module instance is used to store all the information related to an LLVM module.

Class to represent pointers.

static PointerType * get(Type *ElementType, unsigned AddressSpace)

This constructs a pointer to an object of the specified type in a numbered address space.

static PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

The instances of the Type class are immutable: once they are created, they are never changed.

static IntegerType * getInt8Ty(LLVMContext &C)

bool isVoidTy() const

Return true if this is 'void'.

LLVM Value Representation.

iterator_range< user_iterator > users()

LLVMContext & getContext() const

All values hold a context through their type.

self_iterator getIterator()

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

@ Async

The "async continuation" lowering, where each suspend point creates a single continuation function.

@ RetconOnce

The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...

@ Retcon

The "returned-continuation" lowering, where each suspend point creates a single continuation function...

@ Switch

The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...

bool declaresAnyIntrinsic(const Module &M)

bool isSuspendBlock(BasicBlock *BB)

bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)

void suppressCoroAllocs(CoroIdInst *CoroId)

Replaces all @llvm.coro.alloc intrinsics calls associated with a given call @llvm....

void replaceCoroFree(CoroIdInst *CoroId, bool Elide)

This is an optimization pass for GlobalISel generic memory operations.

auto binary_search(R &&Range, T &&Value)

Provide wrappers to std::binary_search which take ranges instead of having to pass begin/end explicit...

decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)

void report_fatal_error(Error Err, bool gen_crash_diag=true)

Report a serious error, calling any installed error handler.

unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)

Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...

raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.

CallInst * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)

Function * ResumePrototype

ArrayRef< Type * > getRetconResumeTypes() const

SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends

Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const

Allocate memory according to the rules of the active lowering.

void cleanCoroutine(SmallVectorImpl< CoroFrameInst * > &CoroFrames, SmallVectorImpl< CoroSaveInst * > &UnusedCoroSaves)

CoroBeginInst * CoroBegin

ArrayRef< Type * > getRetconResultTypes() const

void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const

Deallocate memory according to the rules of the active lowering.

RetconLoweringStorage RetconLowering

void invalidateCoroutine(Function &F, SmallVectorImpl< CoroFrameInst * > &CoroFrames)

void analyze(Function &F, SmallVectorImpl< CoroFrameInst * > &CoroFrames, SmallVectorImpl< CoroSaveInst * > &UnusedCoroSaves)