CoroSplit.cpp Source File (original) (raw)

72#include

73#include

74#include

75#include <initializer_list>

76#include

78using namespace llvm;

80#define DEBUG_TYPE "coro-split"

92 Builder.SetInsertPoint(CB);

98 AttributeList NewAttributes =

100

102 auto WrapperInvoke =

103 Builder.CreateInvoke(Wrapper, Invoke->getNormalDest(),

104 Invoke->getUnwindDest(), {Awaiter, FramePtr});

105

106 WrapperInvoke->setCallingConv(Invoke->getCallingConv());

107 std::copy(Invoke->bundle_op_info_begin(), Invoke->bundle_op_info_end(),

108 WrapperInvoke->bundle_op_info_begin());

109 WrapperInvoke->setAttributes(NewAttributes);

110 WrapperInvoke->setDebugLoc(Invoke->getDebugLoc());

111 NewCall = WrapperInvoke;

113 auto WrapperCall = Builder.CreateCall(Wrapper, {Awaiter, FramePtr});

114

115 WrapperCall->setAttributes(NewAttributes);

116 WrapperCall->setDebugLoc(Call->getDebugLoc());

117 NewCall = WrapperCall;

118 } else {

119 llvm_unreachable("Unexpected coro_await_suspend invocation method");

120 }

121

123 Intrinsic::coro_await_suspend_handle) {

124

125

127

128 Builder.SetInsertPoint(Invoke->getNormalDest()->getFirstInsertionPt());

129 }

130

133 &*Builder.GetInsertPoint());

134

138 auto *ResumeCall = Builder.CreateCall(ResumeTy, ResumeAddr, {NewCall});

140

141

142

144

145 NewCall = ResumeCall;

146 }

147

150}

151

157

167

168

169

170

173

175 if (!EndAsync) {

176 Builder.CreateRetVoid();

177 return true ;

178 }

179

180 auto *MustTailCallFunc = EndAsync->getMustTailCallFunction();

181 if (!MustTailCallFunc) {

182 Builder.CreateRetVoid();

183 return true ;

184 }

185

186

187 auto *CoroEndBlock = End->getParent();

188 auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor();

189 assert(MustTailCallFuncBlock && "Must have a single predecessor block");

190 auto It = MustTailCallFuncBlock->getTerminator()->getIterator();

191 auto *MustTailCall = cast(&*std::prev(It));

192 CoroEndBlock->splice(End->getIterator(), MustTailCallFuncBlock,

193 MustTailCall->getIterator());

194

195

196 Builder.SetInsertPoint(End);

197 Builder.CreateRetVoid();

199

200

202 BB->splitBasicBlock(End);

203 BB->getTerminator()->eraseFromParent();

204

205 auto InlineRes = InlineFunction(*MustTailCall, FnInfo);

206 assert(InlineRes.isSuccess() && "Expected inlining to succeed");

207 (void)InlineRes;

208

209

210 return false;

211}

212

213

217

219

220

221 switch (Shape.ABI) {

222

225 "switch coroutine should not return any values");

226

227

228 if (InRamp)

229 return;

230 Builder.CreateRetVoid();

231 break;

232

233

236 if (!CoroEndBlockNeedsCleanup)

237 return;

238 break;

239 }

240

241

242

247

248 if (!CoroEnd->hasResults()) {

249 assert(RetTy->isVoidTy());

250 Builder.CreateRetVoid();

251 break;

252 }

253

254 auto *CoroResults = CoroEnd->getResults();

255 unsigned NumReturns = CoroResults->numReturns();

256

258 assert(RetStructTy->getNumElements() == NumReturns &&

259 "numbers of returns should match resume function singature");

261 unsigned Idx = 0;

262 for (Value *RetValEl : CoroResults->return_values())

263 ReturnValue = Builder.CreateInsertValue(ReturnValue, RetValEl, Idx++);

264 Builder.CreateRet(ReturnValue);

265 } else if (NumReturns == 0) {

266 assert(RetTy->isVoidTy());

267 Builder.CreateRetVoid();

268 } else {

269 assert(NumReturns == 1);

270 Builder.CreateRet(*CoroResults->retval_begin());

271 }

272 CoroResults->replaceAllUsesWith(

274 CoroResults->eraseFromParent();

275 break;

276 }

277

278

279

282 "retcon coroutine should not return any values");

287 cast(RetStructTy ? RetStructTy->getElementType(0) : RetTy);

288

290 if (RetStructTy) {

291 ReturnValue = Builder.CreateInsertValue(PoisonValue::get(RetStructTy),

292 ReturnValue, 0);

293 }

294 Builder.CreateRet(ReturnValue);

295 break;

296 }

297 }

298

299

301 BB->splitBasicBlock(End);

302 BB->getTerminator()->eraseFromParent();

303}

304

305

306

307

308

309

310

311

312

317 "markCoroutineAsDone is only supported for Switch-Resumed ABI for now.");

318 auto *GepIndex = Builder.CreateStructGEP(

320 "ResumeFn.addr");

323 Builder.CreateStore(NullPtr, GepIndex);

324

325

326

327

328

329

330

331

332

336 "The final suspend should only live in the last position of "

337 "CoroSuspends.");

339 auto *FinalIndex = Builder.CreateStructGEP(

341

342 Builder.CreateStore(IndexVal, FinalIndex);

343 }

344}

345

346

350

351 switch (Shape.ABI) {

352

354

355

356

357

358

359

361 if (InRamp)

362 return;

363 break;

364 }

365

367 break;

368

372 break;

373 }

374

375

378 auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);

379 End->getParent()->splitBasicBlock(End);

380 CleanupRet->getParent()->getTerminator()->eraseFromParent();

381 }

382}

383

392

393

394

395

396

397

398

399

400

401

404 Shape.SwitchLowering.HasFinalSuspend);

405

407 return;

408

410 auto FinalCaseIt = std::prev(Switch->case_end());

411 BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor();

412 Switch->removeCase(FinalCaseIt);

417

418 if (NewF->isCoroOnlyDestroyWhenComplete()) {

419

420

421 Builder.CreateBr(ResumeBB);

422 } else {

423 auto *GepIndex = Builder.CreateStructGEP(

425 "ResumeFn.addr");

426 auto *Load =

427 Builder.CreateLoad(Shape.getSwitchResumePointerType(), GepIndex);

429 Builder.CreateCondBr(Cond, ResumeBB, NewSwitchBB);

430 }

432 }

433}

434

439 auto &Context = Suspend->getParent()->getParent()->getContext();

442}

443

445 const Twine &Suffix,

452

455 OrigF.getName() + Suffix);

456

457 M->getFunctionList().insert(InsertBefore, NewF);

458

459 return NewF;

460}

461

462

463

464

465

469

471 if (NewS->use_empty())

472 return;

473

474

475

477

479 for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()),

480 E = NewF->arg_end();

481 I != E; ++I)

482 Args.push_back(&*I);

483

484

485

487 assert(Args.size() == 1);

488 NewS->replaceAllUsesWith(Args.front());

489 return;

490 }

491

492

495 if (!EVI || EVI->getNumIndices() != 1)

496 continue;

497

498 EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]);

499 EVI->eraseFromParent();

500 }

501

502

503 if (NewS->use_empty())

504 return;

505

506

509 Aggr = Builder.CreateInsertValue(Aggr, Arg, Idx);

510

511 NewS->replaceAllUsesWith(Aggr);

512}

513

515 Value *SuspendResult;

516

517 switch (Shape.ABI) {

518

519

520

521

522

523

526 break;

527

528

530 return;

531

532

533

534

537 return;

538 }

539

541

543 continue;

544

546 MappedCS->replaceAllUsesWith(SuspendResult);

547 MappedCS->eraseFromParent();

548 }

549}

550

559

561 auto &Ctx = OrigF.getContext();

562 for (auto *II : Shape.CoroIsInRampInsts) {

565 NewII->eraseFromParent();

566 }

567}

568

572 return;

573 Value *CachedSlot = nullptr;

574 auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * {

575 if (CachedSlot)

576 return CachedSlot;

577

578

579 for (auto &Arg : F.args()) {

580 if (Arg.isSwiftError()) {

581 CachedSlot = &Arg;

582 return &Arg;

583 }

584 }

585

586

588 F.getEntryBlock().getFirstNonPHIOrDbg());

589 auto Alloca = Builder.CreateAlloca(ValueTy);

590 Alloca->setSwiftError(true);

591

592 CachedSlot = Alloca;

593 return Alloca;

594 };

595

599

600

601 Value *MappedResult;

602 if (Op->arg_empty()) {

603 auto ValueTy = Op->getType();

604 auto Slot = getSwiftErrorSlot(ValueTy);

605 MappedResult = Builder.CreateLoad(ValueTy, Slot);

606 } else {

608 auto Value = MappedOp->getArgOperand(0);

610 auto Slot = getSwiftErrorSlot(ValueTy);

611 Builder.CreateStore(Value, Slot);

612 MappedResult = Slot;

613 }

614

616 MappedOp->eraseFromParent();

617 }

618

619

620 if (VMap == nullptr) {

622 }

623}

624

625

631 DbgVariableRecords.push_back(&DVR);

632 }

633 return DbgVariableRecords;

634}

635

639

643

644

645 bool UseEntryValue = OrigF.getParent()->getTargetTriple().isArch64Bit();

648

649

650

652 auto IsUnreachableBlock = [&](BasicBlock *BB) {

654 &DomTree);

655 };

657 if (IsUnreachableBlock(DVI->getParent()))

658 DVI->eraseFromParent();

660

661 unsigned Uses = 0;

662 for (auto *User : DVI->getVariableLocationOp(0)->users())

667 DVI->eraseFromParent();

668 }

669 };

670 for_each(DbgVariableRecords, RemoveOne);

671}

672

674

675

676

677

678

680 auto *OldEntry = &NewF->getEntryBlock();

681 Entry->setName("entry" + Suffix);

682 Entry->moveBefore(OldEntry);

683 Entry->getTerminator()->eraseFromParent();

684

685

686

687

688 assert(Entry->hasOneUse());

690 assert(BranchToEntry->isUnconditional());

691 Builder.SetInsertPoint(BranchToEntry);

692 Builder.CreateUnreachable();

693 BranchToEntry->eraseFromParent();

694

695

696 Builder.SetInsertPoint(Entry);

697 switch (Shape.ABI) {

699

700

701 auto *SwitchBB =

703 Builder.CreateBr(SwitchBB);

704 SwitchBB->moveAfter(Entry);

705 break;

706 }

710

711

712

720 assert(Branch->isUnconditional());

721 Builder.CreateBr(Branch->getSuccessor(0));

722 break;

723 }

724 }

725

726

727

728 Function *F = OldEntry->getParent();

732 if (!Alloca || I.use_empty())

733 continue;

736 continue;

737 I.moveBefore(*Entry, Entry->getFirstInsertionPt());

738 }

739}

740

741

743

744

745 switch (Shape.ABI) {

746

748 return &*NewF->arg_begin();

749

750

751

752

753

756 auto ContextIdx = ActiveAsyncSuspend->getStorageArgumentIndex() & 0xff;

757 auto *CalleeContext = NewF->getArg(ContextIdx);

758 auto *ProjectionFunc =

759 ActiveAsyncSuspend->getAsyncContextProjectionFunction();

760 auto DbgLoc =

762

763 auto *CallerContext = Builder.CreateCall(ProjectionFunc->getFunctionType(),

764 ProjectionFunc, CalleeContext);

765 CallerContext->setCallingConv(ProjectionFunc->getCallingConv());

766 CallerContext->setDebugLoc(DbgLoc);

767

768 auto &Context = Builder.getContext();

769 auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(

771 Shape.AsyncLowering.FrameOffset, "async.ctx.frameptr");

772

775 assert(InlineRes.isSuccess());

776 (void)InlineRes;

777 return FramePtrAddr;

778 }

779

784

785

786 if (Shape.RetconLowering.IsFrameInlineInStorage)

787 return NewStorage;

788

789

790 return Builder.CreateLoad(FramePtrTy, NewStorage);

791 }

792 }

794}

795

796

797

798

799

800

801

804 if (!ActiveSuspend)

805 return;

806

807

810 if (SPToUpdate.getFile() == DL->getFile())

811 SPToUpdate.setScopeLine(DL->getLine());

812 return;

813 }

814

816

817

819 Branch && Branch->isUnconditional())

820 Successor = Branch->getSuccessor(0)->getFirstNonPHIOrDbg();

821

822

823

828 if ( || DL.getLine() == 0)

829 continue;

830

831 if (SPToUpdate.getFile() == DL->getFile()) {

832 SPToUpdate.setScopeLine(DL.getLine());

833 return;

834 }

835

836 break;

837 }

838

839

841 if (SPToUpdate.getFile() == DL->getFile())

842 SPToUpdate.setScopeLine(DL->getLine());

843}

844

847 Align Alignment, bool NoAlias) {

848 AttrBuilder ParamAttrs(Context);

849 ParamAttrs.addAttribute(Attribute::NonNull);

850 ParamAttrs.addAttribute(Attribute::NoUndef);

851

852 if (NoAlias)

853 ParamAttrs.addAttribute(Attribute::NoAlias);

854

855 ParamAttrs.addAlignmentAttr(Alignment);

856 ParamAttrs.addDereferenceableAttr(Size);

857 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);

858}

859

861 unsigned ParamIndex) {

862 AttrBuilder ParamAttrs(Context);

863 ParamAttrs.addAttribute(Attribute::SwiftAsync);

864 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);

865}

866

868 unsigned ParamIndex) {

869 AttrBuilder ParamAttrs(Context);

870 ParamAttrs.addAttribute(Attribute::SwiftSelf);

871 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);

872}

873

874

875

878

879

880

881

882

883

888 }

889

891

892

893

894 auto savedVisibility = NewF->getVisibility();

895 auto savedUnnamedAddr = NewF->getUnnamedAddr();

896 auto savedDLLStorageClass = NewF->getDLLStorageClass();

897

898

899

900

901 auto savedLinkage = NewF->getLinkage();

903

906

907 auto &Context = NewF->getContext();

908

910 assert(SP != OrigF.getSubprogram() && SP->isDistinct());

912

913

914

916 SP->replaceLinkageName(NewLinkageName);

917 if (DISubprogram *Decl = SP->getDeclaration()) {

918 TempDISubprogram NewDecl = Decl->clone();

919 NewDecl->replaceLinkageName(NewLinkageName);

921 }

922 }

923

924 NewF->setLinkage(savedLinkage);

925 NewF->setVisibility(savedVisibility);

926 NewF->setUnnamedAddr(savedUnnamedAddr);

927 NewF->setDLLStorageClass(savedDLLStorageClass);

928

929

930

932 NewF->hasMetadata(LLVMContext::MD_func_sanitize))

933 NewF->eraseMetadata(LLVMContext::MD_func_sanitize);

934

935

936 auto OrigAttrs = NewF->getAttributes();

937 auto NewAttrs = AttributeList();

938

939 switch (Shape.ABI) {

941

942

943 NewAttrs = NewAttrs.addFnAttributes(

944 Context, AttrBuilder(Context, OrigAttrs.getFnAttrs()));

945

947 Shape.FrameAlign, false);

948 break;

951 if (OrigF.hasParamAttribute(Shape.AsyncLowering.ContextArgNo,

952 Attribute::SwiftAsync)) {

953 uint32_t ArgAttributeIndices =

954 ActiveAsyncSuspend->getStorageArgumentIndex();

955 auto ContextArgIndex = ArgAttributeIndices & 0xff;

957

958

959

960 auto SwiftSelfIndex = ArgAttributeIndices >> 8;

961 if (SwiftSelfIndex)

963 }

964

965

966 auto FnAttrs = OrigF.getAttributes().getFnAttrs();

967 NewAttrs = NewAttrs.addFnAttributes(Context, AttrBuilder(Context, FnAttrs));

968 break;

969 }

972

973

974 NewAttrs = Shape.RetconLowering.ResumePrototype->getAttributes();

975

976

978 Shape.getRetconCoroId()->getStorageSize(),

979 Shape.getRetconCoroId()->getStorageAlignment(),

980 true);

981

982 break;

983 }

984

985 switch (Shape.ABI) {

986

987

988

989

992

995 break;

996

997

998

999

1001 break;

1002

1003

1004

1005

1007 break;

1008 }

1009

1010 NewF->setAttributes(NewAttrs);

1011 NewF->setCallingConv(Shape.getResumeFunctionCC());

1012

1013

1015

1016

1017 for (CallInst *ResumeCall : Shape.SymmetricTransfers) {

1019 if (TTI.supportsTailCallFor(ResumeCall)) {

1020

1021

1023 }

1024

1025

1026

1030 Builder.CreateRetVoid();

1032 }

1033

1034 Builder.SetInsertPoint(&NewF->getEntryBlock().front());

1036

1037

1041

1042

1043 auto *NewVFrame = Builder.CreateBitCast(

1046 if (OldVFrame != NewVFrame)

1048

1049

1050

1051 for (Instruction *DummyArg : DummyArgs) {

1052 DummyArg->replaceAllUsesWith(PoisonValue::get(DummyArg->getType()));

1053 DummyArg->deleteValue();

1054 }

1055

1056 switch (Shape.ABI) {

1058

1059

1060

1061 if (Shape.SwitchLowering.HasFinalSuspend)

1063 break;

1067

1068

1070 "no active suspend when lowering a continuation-style coroutine");

1072 break;

1073 }

1074

1075

1077

1078

1080

1081

1083

1085

1086

1088}

1089

1091

1094

1095

1097

1098

1099

1102}

1103

1106

1109 auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0);

1110 auto *OrigContextSize = FuncPtrStruct->getOperand(1);

1111 auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(),

1114 FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize);

1115

1117}

1118

1120

1121 auto *SizeIntrin = Shape.CoroSizes.back();

1122 Module *M = SizeIntrin->getModule();

1124 return DL.getTypeAllocSize(Shape.FrameTy);

1125}

1126

1130

1135 }

1136

1138 return;

1139

1140

1141 auto *SizeIntrin = Shape.CoroSizes.back();

1142 auto *SizeConstant =

1144

1148 }

1149}

1150

1153

1154#ifndef NDEBUG

1155

1156

1157

1160#endif

1161}

1162

1163

1164

1166 auto *CoroBegin = Shape.CoroBegin;

1167 switch (Shape.ABI) {

1172 if (AllocInst) {

1174 auto *Frame = Builder.CreateAlloca(Shape.FrameTy);

1175 Frame->setAlignment(Shape.FrameAlign);

1176 AllocInst->replaceAllUsesWith(Builder.getFalse());

1177 AllocInst->eraseFromParent();

1178 CoroBegin->replaceAllUsesWith(Frame);

1179 } else {

1180 CoroBegin->replaceAllUsesWith(CoroBegin->getMem());

1181 }

1182

1183 break;

1184 }

1188 CoroBegin->replaceAllUsesWith(PoisonValue::get(CoroBegin->getType()));

1189 break;

1190 }

1191

1194}

1195

1196

1197

1198

1201

1203 continue;

1204

1206 return true;

1207 }

1208 return false;

1209}

1210

1214

1215 Set.insert(SaveBB);

1217

1218

1219

1220

1221 while (!Worklist.empty()) {

1223 Set.insert(BB);

1225 if (!Set.contains(Pred))

1227 }

1228

1229

1230 Set.erase(SaveBB);

1231 Set.erase(ResDesBB);

1232

1233 for (auto *BB : Set)

1235 return true;

1236

1237 return false;

1238}

1239

1241 auto *SaveBB = Save->getParent();

1242 auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();

1245

1246 if (SaveBB == ResumeOrDestroyBB)

1248

1249

1251 return true;

1252

1253

1255 {ResumeOrDestroyBB->getFirstNonPHIIt(), ResumeOrDestroyIt}))

1256 return true;

1257

1258

1260 return true;

1261

1262 return false;

1263}

1264

1265

1266

1270 if (!Prev) {

1271 auto *Pred = Suspend->getParent()->getSinglePredecessor();

1272 if (!Pred)

1273 return false;

1274 Prev = Pred->getTerminator();

1275 }

1276

1278 if (!CB)

1279 return false;

1280

1282

1283

1285 if (!SubFn)

1286 return false;

1287

1288

1289 if (SubFn->getFrame() != CoroBegin)

1290 return false;

1291

1292

1293

1294

1297 return false;

1298

1299

1300

1303 Save->eraseFromParent();

1304

1305

1308 }

1309

1310

1313

1314

1315 if (CalledValue != SubFn && CalledValue->user_empty())

1317 I->eraseFromParent();

1318

1319

1320 if (SubFn->user_empty())

1321 SubFn->eraseFromParent();

1322

1323 return true;

1324}

1325

1326

1328

1330 return;

1331

1333 size_t I = 0, N = S.size();

1334 if (N == 0)

1335 return;

1336

1337 size_t ChangedFinalIndex = std::numeric_limits<size_t>::max();

1338 while (true) {

1340

1341

1343 if (--N == I)

1344 break;

1345

1347

1350 ChangedFinalIndex = I;

1351 }

1352

1353 continue;

1354 }

1355 if (++I == N)

1356 break;

1357 }

1358 S.resize(N);

1359

1360

1361

1362 if (ChangedFinalIndex < N) {

1364 std::swap(S[ChangedFinalIndex], S.back());

1365 }

1366}

1367

1368namespace {

1369

1370struct SwitchCoroutineSplitter {

1371 static void split(Function &F, coro::Shape &Shape,

1372 SmallVectorImpl<Function *> &Clones,

1373 TargetTransformInfo &TTI) {

1374 assert(Shape.ABI == coro::ABI::Switch);

1375

1376

1377

1378

1379 createResumeEntryBlock(F, Shape);

1381 F, ".resume", Shape, coro::CloneKind::SwitchResume, TTI);

1383 F, ".destroy", Shape, coro::CloneKind::SwitchUnwind, TTI);

1385 F, ".cleanup", Shape, coro::CloneKind::SwitchCleanup, TTI);

1386

1390

1391

1392 updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);

1393

1398

1399

1400

1401

1402 setCoroInfo(F, Shape, Clones);

1403 }

1404

1405

1406

1407

1408

1409

1410

1411

1412 static Function *createNoAllocVariant(Function &F, coro::Shape &Shape,

1413 SmallVectorImpl<Function *> &Clones) {

1414 assert(Shape.ABI == coro::ABI::Switch);

1415 auto *OrigFnTy = F.getFunctionType();

1416 auto OldParams = OrigFnTy->params();

1417

1419 NewParams.reserve(OldParams.size() + 1);

1420 NewParams.append(OldParams.begin(), OldParams.end());

1422

1423 auto *NewFnTy = FunctionType::get(OrigFnTy->getReturnType(), NewParams,

1424 OrigFnTy->isVarArg());

1426 Function::Create(NewFnTy, F.getLinkage(), F.getName() + ".noalloc");

1427

1429 unsigned int Idx = 0;

1430 for (const auto &I : F.args()) {

1431 VMap[&I] = NoAllocF->getArg(Idx++);

1432 }

1433

1434

1435 auto FrameIdx = NoAllocF->arg_size() - 1;

1438 CloneFunctionChangeType::LocalChangesOnly, Returns);

1439

1441 auto *NewCoroBegin =

1446 NewCoroBegin->replaceAllUsesWith(NoAllocF->getArg(FrameIdx));

1447 NewCoroBegin->eraseFromParent();

1448 }

1449

1451 M->getFunctionList().insert(M->end(), NoAllocF);

1452

1455

1456

1459 false);

1460

1462

1464

1465

1466 setCoroInfo(F, Shape, Clones);

1467

1468

1469

1471 return NoAllocF;

1472 }

1473

1474private:

1475

1476

1477 static void createResumeEntryBlock(Function &F, coro::Shape &Shape) {

1479

1480 DIBuilder DBuilder(*F.getParent(), false);

1481 DISubprogram *DIS = F.getSubprogram();

1482

1483

1484

1485 bool AddDebugLabels = DIS && DIS->getUnit() &&

1486 (DIS->getUnit()->getEmissionKind() ==

1487 DICompileUnit::DebugEmissionKind::FullDebug);

1488

1489

1490

1491

1492

1493

1494

1495

1496

1497

1500

1503 auto *FrameTy = Shape.FrameTy;

1504 auto *GepIndex = Builder.CreateStructGEP(

1506 auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");

1508 Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());

1510

1511

1512 size_t SuspendIndex = 0;

1515 ConstantInt *IndexVal = Shape.getIndex(SuspendIndex);

1516

1517

1518

1519

1520 auto *Save = S->getCoroSave();

1521 Builder.SetInsertPoint(Save);

1522 if (S->isFinal()) {

1523

1524

1526 } else {

1527 auto *GepIndex = Builder.CreateStructGEP(

1529 Builder.CreateStore(IndexVal, GepIndex);

1530 }

1531

1533 Save->eraseFromParent();

1534

1535

1536

1537

1538

1539

1540

1541

1542

1543

1544

1545

1546

1547

1548

1549

1550

1551

1552

1553

1554

1555

1556

1557

1558

1559 auto *SuspendBB = S->getParent();

1560 auto *ResumeBB =

1561 SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex));

1562 auto *LandingBB = ResumeBB->splitBasicBlock(

1563 S->getNextNode(), ResumeBB->getName() + Twine(".landing"));

1564 Switch->addCase(IndexVal, ResumeBB);

1565

1566 cast(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);

1568 PN->insertBefore(LandingBB->begin());

1569 S->replaceAllUsesWith(PN);

1570 PN->addIncoming(Builder.getInt8(-1), SuspendBB);

1571 PN->addIncoming(S, ResumeBB);

1572

1573 if (AddDebugLabels) {

1574 if (DebugLoc SuspendLoc = S->getDebugLoc()) {

1575 std::string LabelName =

1576 ("__coro_resume_" + Twine(SuspendIndex)).str();

1577

1578

1579

1580

1581

1582 DILocation *DILoc = SuspendLoc;

1583 while (DILocation *InlinedAt = DILoc->getInlinedAt())

1584 DILoc = InlinedAt;

1585

1586 DILabel *ResumeLabel =

1587 DBuilder.createLabel(DIS, LabelName, DILoc->getFile(),

1588 SuspendLoc.getLine(), SuspendLoc.getCol(),

1589 true,

1590 SuspendIndex,

1591 false);

1592 DBuilder.insertLabel(ResumeLabel, DILoc, ResumeBB->begin());

1593 }

1594 }

1595

1596 ++SuspendIndex;

1597 }

1598

1599 Builder.SetInsertPoint(UnreachBB);

1600 Builder.CreateUnreachable();

1601 DBuilder.finalize();

1602

1604 }

1605

1606

1607 static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,

1608 Function *DestroyFn, Function *CleanupFn) {

1610

1611 auto *ResumeAddr = Builder.CreateStructGEP(

1613 "resume.addr");

1614 Builder.CreateStore(ResumeFn, ResumeAddr);

1615

1616 Value *DestroyOrCleanupFn = DestroyFn;

1617

1619 if (CoroAllocInst *CA = CoroId->getCoroAlloc()) {

1620

1621

1622 DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);

1623 }

1624

1625 auto *DestroyAddr = Builder.CreateStructGEP(

1627 "destroy.addr");

1628 Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);

1629 }

1630

1631

1632

1633

1634

1635

1636

1637

1638

1639

1640

1641

1642

1643

1644 static void setCoroInfo(Function &F, coro::Shape &Shape,

1646

1647

1652 auto *ArrTy = ArrayType::get(Part->getType(), Args.size());

1653

1655 auto *GV = new GlobalVariable(*M, ConstVal->getType(), true,

1656 GlobalVariable::PrivateLinkage, ConstVal,

1657 F.getName() + Twine(".resumers"));

1658

1659

1660 LLVMContext &C = F.getContext();

1663 }

1664};

1665

1666}

1667

1671 auto &Context = Suspend->getParent()->getParent()->getContext();

1673

1675 auto *Val = Builder.CreateBitOrPointerCast(Continuation, Int8PtrTy);

1676 ResumeIntrinsic->replaceAllUsesWith(Val);

1677 ResumeIntrinsic->eraseFromParent();

1680}

1681

1682

1686 size_t ArgIdx = 0;

1687 for (auto *paramTy : FnTy->params()) {

1689 if (paramTy != FnArgs[ArgIdx]->getType())

1691 Builder.CreateBitOrPointerCast(FnArgs[ArgIdx], paramTy));

1692 else

1693 CallArgs.push_back(FnArgs[ArgIdx]);

1694 ++ArgIdx;

1695 }

1696}

1697

1703

1704

1707

1708 auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs);

1709

1710 if (TTI.supportsTailCallFor(TailCall)) {

1712 }

1713 TailCall->setDebugLoc(Loc);

1714 TailCall->setCallingConv(MustTailCallFn->getCallingConv());

1715 return TailCall;

1716}

1717

1723

1724

1725 F.removeFnAttr(Attribute::NoReturn);

1726 F.removeRetAttr(Attribute::NoAlias);

1727 F.removeRetAttr(Attribute::NonNull);

1728

1729 auto &Context = F.getContext();

1731

1732 auto *Id = Shape.getAsyncCoroId();

1734

1735 auto *FramePtr = Id->getStorage();

1736 FramePtr = Builder.CreateBitOrPointerCast(FramePtr, Int8PtrTy);

1737 FramePtr = Builder.CreateConstInBoundsGEP1_32(

1739 "async.ctx.frameptr");

1740

1741

1742 {

1743

1747 }

1748

1749

1750 auto NextF = std::next(F.getIterator());

1751

1752

1756

1757

1758 auto ResumeNameSuffix = ".resume.";

1759 auto ProjectionFunctionName =

1760 Suspend->getAsyncContextProjectionFunction()->getName();

1761 bool UseSwiftMangling = false;

1762 if (ProjectionFunctionName == "__swift_async_resume_project_context") {

1763 ResumeNameSuffix = "TQ";

1764 UseSwiftMangling = true;

1765 } else if (ProjectionFunctionName == "__swift_async_resume_get_context") {

1766 ResumeNameSuffix = "TY";

1767 UseSwiftMangling = true;

1768 }

1771 UseSwiftMangling ? ResumeNameSuffix + Twine(Idx) + "_"

1772 : ResumeNameSuffix + Twine(Idx),

1773 NextF, Suspend);

1775

1776

1777

1778 auto *SuspendBB = Suspend->getParent();

1779 auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);

1780 auto *Branch = cast(SuspendBB->getTerminator());

1781

1782

1783 auto *ReturnBB =

1785 Branch->setSuccessor(0, ReturnBB);

1786

1788

1789

1790 auto *Fn = Suspend->getMustTailCallFunction();

1795 FnArgs, Builder);

1796 Builder.CreateRetVoid();

1799

1800

1802 }

1803

1805

1807 auto *Suspend = CS;

1808 auto *Clone = Clones[Idx];

1809

1811 Suspend, TTI);

1812 }

1813}

1814

1820

1821

1822

1823 F.removeFnAttr(Attribute::NoReturn);

1824 F.removeRetAttr(Attribute::NoAlias);

1825 F.removeRetAttr(Attribute::NonNull);

1826

1827

1828 auto *Id = Shape.getRetconCoroId();

1829 Value *RawFramePtr;

1830 if (Shape.RetconLowering.IsFrameInlineInStorage) {

1831 RawFramePtr = Id->getStorage();

1832 } else {

1834

1835

1837 auto Size = DL.getTypeAllocSize(Shape.FrameTy);

1838

1839

1840

1841

1842 RawFramePtr = Shape.emitAlloc(Builder, Builder.getInt64(Size), nullptr);

1843 RawFramePtr =

1844 Builder.CreateBitCast(RawFramePtr, Shape.CoroBegin->getType());

1845

1846

1847 Builder.CreateStore(RawFramePtr, Id->getStorage());

1848 }

1849

1850

1851 {

1852

1854 Shape.CoroBegin->replaceAllUsesWith(RawFramePtr);

1856 }

1857

1858

1860 PHINode *ContinuationPhi = nullptr;

1862

1863

1864 auto NextF = std::next(F.getIterator());

1865

1866

1870

1871

1873 F, Shape, ".resume." + Twine(Idx), NextF, nullptr);

1875

1876

1877

1878 auto SuspendBB = Suspend->getParent();

1879 auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);

1881

1882

1883 if (!ReturnBB) {

1884

1885 ReturnBB =

1887 Shape.RetconLowering.ReturnBlock = ReturnBB;

1888

1890

1891

1892 ContinuationPhi =

1893 Builder.CreatePHI(Continuation->getType(), Shape.CoroSuspends.size());

1894

1895

1897

1898

1899 for (auto *ResultTy : Shape.getRetconResultTypes())

1901 Builder.CreatePHI(ResultTy, Shape.CoroSuspends.size()));

1902

1903

1904 auto RetTy = F.getReturnType();

1905

1906

1907

1908

1909 auto CastedContinuationTy =

1910 (ReturnPHIs.empty() ? RetTy : RetTy->getStructElementType(0));

1911 auto *CastedContinuation =

1912 Builder.CreateBitCast(ContinuationPhi, CastedContinuationTy);

1913

1914 Value *RetV = CastedContinuation;

1915 if (!ReturnPHIs.empty()) {

1916 auto ValueIdx = 0;

1918 RetV = Builder.CreateInsertValue(RetV, CastedContinuation, ValueIdx++);

1919

1920 for (auto Phi : ReturnPHIs)

1921 RetV = Builder.CreateInsertValue(RetV, Phi, ValueIdx++);

1922 }

1923

1924 Builder.CreateRet(RetV);

1925 }

1926

1927

1928 Branch->setSuccessor(0, ReturnBB);

1929 assert(ContinuationPhi);

1931 for (auto [Phi, VUse] :

1933 Phi->addIncoming(VUse, SuspendBB);

1934 }

1935

1937

1939 auto Suspend = CS;

1940 auto Clone = Clones[Idx];

1941

1943 Suspend, TTI);

1944 }

1945}

1946

1947namespace {

1950

1951public:

1952 PrettyStackTraceFunction(Function &F) : F(F) {}

1954 OS << "While splitting coroutine ";

1955 F.printAsOperand(OS, false, F.getParent());

1956 OS << "\n";

1957 }

1958};

1959}

1960

1961

1964 for (auto *End : Shape.CoroEnds) {

1966 }

1967 } else {

1970 }

1971}

1972

1975 auto &Ctx = II->getContext();

1977 II->eraseFromParent();

1978 }

1979}

1980

1982 for (auto *U : F.users()) {

1984 auto *Caller = CB->getFunction();

1985 if (Caller && Caller->isPresplitCoroutine() &&

1986 CB->hasFnAttr(llvm::Attribute::CoroElideSafe))

1987 return true;

1988 }

1989 }

1990 return false;

1991}

1992

1996 SwitchCoroutineSplitter::split(F, Shape, Clones, TTI);

1997}

1998

2001 bool OptimizeFrame) {

2002 PrettyStackTraceFunction prettyStackTrace(F);

2003

2004 auto &Shape = ABI.Shape;

2006

2008

2010

2012 ABI.buildCoroutineFrame(OptimizeFrame);

2014

2015 bool isNoSuspendCoroutine = Shape.CoroSuspends.empty();

2016

2017 bool shouldCreateNoAllocVariant =

2020

2021

2022

2023 if (isNoSuspendCoroutine) {

2025 } else {

2026 ABI.splitCoroutine(F, Shape, Clones, TTI);

2027 }

2028

2029

2030

2032

2033

2034

2035

2040

2043

2044 if (shouldCreateNoAllocVariant)

2045 SwitchCoroutineSplitter::createNoAllocVariant(F, Shape, Clones);

2046}

2047

2053

2054 auto *CurrentSCC = &C;

2055 if (!Clones.empty()) {

2056 switch (Shape.ABI) {

2058

2059

2060 for (Function *Clone : Clones)

2062 break;

2066

2067

2068 if (!Clones.empty())

2070 break;

2071 }

2072

2073

2075 UR, FAM);

2076 }

2077

2078

2079

2082 AM, UR, FAM);

2083 return *CurrentSCC;

2084}

2085

2086

2089 auto CastFn = Prepare->getArgOperand(0);

2091

2092

2093

2094

2095

2096

2097

2099

2101 if (!Cast || Cast->getType() != Fn->getType())

2102 continue;

2103

2104

2105 Cast->replaceAllUsesWith(Fn);

2106 Cast->eraseFromParent();

2107 }

2108

2109

2110

2113

2114

2116 if (!Cast->use_empty())

2117 break;

2118 CastFn = Cast->getOperand(0);

2119 Cast->eraseFromParent();

2120 }

2121}

2122

2135

2139 auto *PrepareFn = M.getFunction(Name);

2140 if (PrepareFn && !PrepareFn->use_empty())

2142}

2143

2144static std::unique_ptrcoro::BaseABI

2146 std::function<bool(Instruction &)> IsMatCallback,

2150 if (CustomABI >= GenCustomABIs.size())

2151 llvm_unreachable("Custom ABI not found amoung those specified");

2152 return GenCustomABIs[CustomABI](F, S);

2153 }

2154

2155 switch (S.ABI) {

2157 return std::make_uniquecoro::SwitchABI(F, S, IsMatCallback);

2159 return std::make_uniquecoro::AsyncABI(F, S, IsMatCallback);

2161 return std::make_uniquecoro::AnyRetconABI(F, S, IsMatCallback);

2163 return std::make_uniquecoro::AnyRetconABI(F, S, IsMatCallback);

2164 }

2166}

2167

2170 std::unique_ptrcoro::BaseABI ABI =

2172 ABI->init();

2173 return ABI;

2174 }),

2175 OptimizeFrame(OptimizeFrame) {}

2176

2180 std::unique_ptrcoro::BaseABI ABI =

2182 ABI->init();

2183 return ABI;

2184 }),

2185 OptimizeFrame(OptimizeFrame) {}

2186

2187

2188

2192 std::unique_ptrcoro::BaseABI ABI =

2194 ABI->init();

2195 return ABI;

2196 }),

2197 OptimizeFrame(OptimizeFrame) {}

2198

2199

2200

2202 std::function<bool(Instruction &)> IsMatCallback,

2205 std::unique_ptrcoro::BaseABI ABI =

2206 CreateNewABI(F, S, IsMatCallback, GenCustomABIs);

2207 ABI->init();

2208 return ABI;

2209 }),

2210 OptimizeFrame(OptimizeFrame) {}

2211

2215

2216

2217

2218 Module &M = *C.begin()->getFunction().getParent();

2219 auto &FAM =

2221

2222

2226

2227

2230 if (N.getFunction().isPresplitCoroutine())

2232

2233 if (Coroutines.empty() && PrepareFns.empty())

2235

2236 auto *CurrentSCC = &C;

2237

2240 LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()

2241 << "\n");

2242

2243

2244

2245

2247

2250 continue;

2251

2252 F.setSplittedCoroutine();

2253

2254 std::unique_ptrcoro::BaseABI ABI = CreateAndInitABI(F, Shape);

2255

2260 *N, Shape, Clones, *CurrentSCC, CG, AM, UR, FAM);

2261

2263 ORE.emit([&]() {

2265 << "Split '" << ore::NV("function", F.getName())

2266 << "' (frame_size=" << ore::NV("frame_size", Shape.FrameSize)

2268 });

2269

2271

2272 UR.CWorklist.insert(CurrentSCC);

2273 for (Function *Clone : Clones)

2276

2277

2279 }

2280 }

2281

2282 for (auto *PrepareFn : PrepareFns) {

2284 }

2285

2287}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

amdgpu aa AMDGPU Address space based Alias Analysis Wrapper

AMDGPU Lower Kernel Arguments

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)

Expand Atomic instructions

This file contains the simple types necessary to represent the attributes associated with functions a...

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

This file provides interfaces used to manipulate a call graph, regardless if it is a "old style" Call...

This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...

This file contains the declarations for the subclasses of Constant, which represent the different fla...

static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)

Definition CoroSplit.cpp:867

static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy)

Definition CoroSplit.cpp:1240

static LazyCallGraph::SCC & updateCallGraphAfterCoroutineSplit(LazyCallGraph::Node &N, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, LazyCallGraph::SCC &C, LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)

Definition CoroSplit.cpp:2048

static void replaceFallthroughCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InRamp, CallGraph *CG)

Replace a non-unwind call to llvm.coro.end.

Definition CoroSplit.cpp:214

static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)

Definition CoroSplit.cpp:569

static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InRamp, CallGraph *CG)

Definition CoroSplit.cpp:384

static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)

Definition CoroSplit.cpp:860

static void maybeFreeRetconStorage(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG)

Definition CoroSplit.cpp:158

static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB)

Definition CoroSplit.cpp:1211

static Function * createCloneDeclaration(Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)

Definition CoroSplit.cpp:444

static FunctionType * getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend)

Definition CoroSplit.cpp:436

static void updateScopeLine(Instruction *ActiveSuspend, DISubprogram &SPToUpdate)

Adjust the scope line of the funclet to the first line number after the suspend point.

Definition CoroSplit.cpp:802

static void removeCoroIsInRampFromRampFunction(const coro::Shape &Shape)

Definition CoroSplit.cpp:1973

static void addPrepareFunction(const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)

Definition CoroSplit.cpp:2136

static SmallVector< DbgVariableRecord * > collectDbgVariableRecords(Function &F)

Returns all debug records in F.

Definition CoroSplit.cpp:627

static void simplifySuspendPoints(coro::Shape &Shape)

Definition CoroSplit.cpp:1327

static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment, bool NoAlias)

Definition CoroSplit.cpp:845

static bool hasSafeElideCaller(Function &F)

Definition CoroSplit.cpp:1981

static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)

Definition CoroSplit.cpp:2123

static void replaceFrameSizeAndAlignment(coro::Shape &Shape)

Definition CoroSplit.cpp:1127

static std::unique_ptr< coro::BaseABI > CreateNewABI(Function &F, coro::Shape &S, std::function< bool(Instruction &)> IsMatCallback, const SmallVector< CoroSplitPass::BaseABITy > GenCustomABIs)

Definition CoroSplit.cpp:2145

static bool replaceCoroEndAsync(AnyCoroEndInst *End)

Replace an llvm.coro.end.async.

Definition CoroSplit.cpp:171

static void doSplitCoroutine(Function &F, SmallVectorImpl< Function * > &Clones, coro::BaseABI &ABI, TargetTransformInfo &TTI, bool OptimizeFrame)

Definition CoroSplit.cpp:1999

static bool hasCallsInBlockBetween(iterator_range< BasicBlock::iterator > R)

Definition CoroSplit.cpp:1199

static bool simplifySuspendPoint(CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)

Definition CoroSplit.cpp:1267

static void removeCoroEndsFromRampFunction(const coro::Shape &Shape)

Remove calls to llvm.coro.end in the original function.

Definition CoroSplit.cpp:1962

static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr)

Definition CoroSplit.cpp:313

static void updateAsyncFuncPointerContextSize(coro::Shape &Shape)

Definition CoroSplit.cpp:1104

static void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > &CallArgs)

Coerce the arguments in FnArgs according to FnTy in CallArgs.

Definition CoroSplit.cpp:1683

static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InRamp, CallGraph *CG)

Replace an unwind call to llvm.coro.end.

Definition CoroSplit.cpp:347

static void lowerAwaitSuspend(IRBuilder<> &Builder, CoroAwaitSuspendInst *CB, coro::Shape &Shape)

Definition CoroSplit.cpp:86

static void lowerAwaitSuspends(Function &F, coro::Shape &Shape)

Definition CoroSplit.cpp:152

static void handleNoSuspendCoroutine(coro::Shape &Shape)

Definition CoroSplit.cpp:1165

static void postSplitCleanup(Function &F)

Definition CoroSplit.cpp:1151

static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)

Replace a call to llvm.coro.prepare.retcon.

Definition CoroSplit.cpp:2087

static TypeSize getFrameSizeForShape(coro::Shape &Shape)

Definition CoroSplit.cpp:1119

static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend, Value *Continuation)

Definition CoroSplit.cpp:1668

This file defines the DenseMap class.

This file contains constants used for implementing Dwarf debug support.

This file provides various utilities for inspecting and working with the control flow graph in LLVM I...

Module.h This file contains the declarations for the Module class.

Implements a lazy call graph analysis and related passes for the new pass manager.

Machine Check Debug Module

uint64_t IntrinsicInst * II

FunctionAnalysisManager FAM

This file provides a priority worklist.

const SmallVectorImpl< MachineOperand > & Cond

Remove Loads Into Fake Uses

This file defines the SmallPtrSet class.

This file defines the SmallVector class.

static SymbolRef::Type getType(const Symbol *Sym)

This pass exposes codegen information to IR-level passes.

static const unsigned FramePtr

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

CoroAllocInst * getCoroAlloc()

This class represents an incoming formal argument to a Function.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

LLVM Basic Block Representation.

const Function * getParent() const

Return the enclosing method, or null if none.

static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)

Creates a new BasicBlock.

LLVM_ABI BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)

Split the basic block into two basic blocks at the specified instruction.

InstListType::iterator iterator

Instruction iterators...

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const

Return an operand bundle by name, if present.

Function * getCalledFunction() const

Returns the function called, or null if this is an indirect function invocation or the function signa...

Value * getCalledOperand() const

Value * getArgOperand(unsigned i) const

AttributeList getAttributes() const

Return the attributes for this call.

The basic data container for the call graph of a Module of IR.

This class represents a function call, abstracting a target machine's calling convention.

static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)

static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)

Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.

This is the shared class of boolean and integer constants.

static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)

static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)

static LLVM_ABI ConstantPointerNull * get(PointerType *T)

Static factory methods - Return objects of the specified value.

static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)

static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)

Return the ConstantTokenNone.

This represents the llvm.coro.align instruction.

This represents the llvm.coro.await.suspend.{void,bool,handle} instructions.

Value * getAwaiter() const

Function * getWrapperFunction() const

This class represents the llvm.coro.begin or llvm.coro.begin.custom.abi instructions.

bool hasCustomABI() const

void setInfo(Constant *C)

This represents the llvm.coro.size instruction.

This represents the llvm.coro.suspend.async instruction.

CoroAsyncResumeInst * getResumeFunction() const

This represents the llvm.coro.suspend instruction.

CoroSaveInst * getCoroSave() const

Subprogram description. Uses SubclassData1.

A parsed version of the target data layout string in and methods for querying it.

Record of a variable value-assignment, aka a non instruction representation of the dbg....

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

LLVM_ABI bool isReachableFromEntry(const Use &U) const

Provide an overload for a Use.

This class represents a freeze function that returns random concrete value if an operand is either a ...

A proxy from a FunctionAnalysisManager to an SCC.

Class to represent function types.

Type * getReturnType() const

static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)

FunctionType * getFunctionType() const

Returns the FunctionType for me.

Intrinsic::ID getIntrinsicID() const LLVM_READONLY

getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...

CallingConv::ID getCallingConv() const

getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...

AttributeList getAttributes() const

Return the attribute list for this Function.

void setAttributes(AttributeList Attrs)

Set the attribute list for this Function.

LLVMContext & getContext() const

getContext - Return a reference to the LLVMContext associated with this function.

Argument * getArg(unsigned i) const

void setLinkage(LinkageTypes LT)

Module * getParent()

Get the module that this global value is contained inside of...

PointerType * getType() const

Global values are always pointers.

@ InternalLinkage

Rename collisions when linking (static functions).

@ ExternalLinkage

Externally visible function.

const Constant * getInitializer() const

getInitializer - Return the initializer for this global variable.

LLVM_ABI void setInitializer(Constant *InitVal)

setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

This class captures the data input to the InlineFunction call, and records the auxiliary results prod...

const DebugLoc & getDebugLoc() const

Return the debug location for this node as a DebugLoc.

LLVM_ABI InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

This is an important class for using LLVM in a threaded context.

A node in the call graph.

An SCC of the call graph.

A lazily constructed view of the call graph of a module.

LLVM_ABI void addSplitFunction(Function &OriginalFunction, Function &NewFunction)

Add a new function split/outlined from an existing function.

LLVM_ABI void addSplitRefRecursiveFunctions(Function &OriginalFunction, ArrayRef< Function * > NewFunctions)

Add new ref-recursive functions split/outlined from an existing function.

Node & get(Function &F)

Get a graph node for a given function, scanning it to populate the graph data as necessary.

SCC * lookupSCC(Node &N) const

Lookup a function's SCC in the graph.

static std::enable_if_t< std::is_base_of< MDNode, T >::value, T * > replaceWithUniqued(std::unique_ptr< T, TempMDNodeDeleter > N)

Replace a temporary node with a uniqued one.

static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)

A Module instance is used to store all the information related to an LLVM module.

FunctionListType::iterator iterator

The Function iterators.

void addIncoming(Value *V, BasicBlock *BB)

Add an incoming value to the end of the PHI list.

static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...

static PointerType * getUnqual(Type *ElementType)

This constructs a pointer to an object of the specified type in the default address space (address sp...

static LLVM_ABI PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

PrettyStackTraceEntry - This class is used to represent a frame of the "pretty" stack trace that is d...

Return a value (possibly void), from a function.

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void reserve(size_type N)

void append(ItTy in_start, ItTy in_end)

Add the specified range to the end of the SmallVector.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

LLVM_ABI Type * getTypeAtIndex(const Value *V) const

Given an index value into the type, return the type of the element.

Analysis pass providing the TargetTransformInfo.

This pass provides access to the codegen interfaces that are needed for IR-level transformations.

Value handle that tracks a Value across RAUW.

ValueTy * getValPtr() const

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI Type * getVoidTy(LLVMContext &C)

static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)

LLVMContext & getContext() const

Return the LLVMContext in which this type was uniqued.

A Use represents the edge between a Value definition and its users.

void setOperand(unsigned i, Value *Val)

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

LLVM_ABI void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

iterator_range< user_iterator > users()

LLVM_ABI const Value * stripPointerCasts() const

Strip off pointer casts, all-zero GEPs and address space casts.

LLVM_ABI LLVMContext & getContext() const

All values hold a context through their type.

iterator_range< use_iterator > uses()

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI) override

Definition CoroSplit.cpp:1815

void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI) override

Definition CoroSplit.cpp:1718

void replaceSwiftErrorOps()

Definition CoroSplit.cpp:636

void salvageDebugInfo()

Definition CoroSplit.cpp:640

AnyCoroSuspendInst * ActiveSuspend

The active suspend instruction; meaningful only for continuation and async ABIs.

Value * deriveNewFramePointer()

Derive the value of the new frame pointer.

Definition CoroSplit.cpp:742

void replaceEntryBlock()

Definition CoroSplit.cpp:673

void replaceCoroSuspends()

Definition CoroSplit.cpp:514

void handleFinalSuspend()

Definition CoroSplit.cpp:402

TargetTransformInfo & TTI

static Function * createClone(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, Function *NewF, AnyCoroSuspendInst *ActiveSuspend, TargetTransformInfo &TTI)

Create a clone for a continuation lowering.

void replaceCoroIsInRamp()

Definition CoroSplit.cpp:560

void replaceCoroEnds()

Definition CoroSplit.cpp:551

bool isSwitchDestroyFunction()

void replaceRetconOrAsyncSuspendUses()

Replace uses of the active llvm.coro.suspend.retcon/async call with the arguments to the continuation...

Definition CoroSplit.cpp:466

virtual void create()

Clone the body of the original function into a resume function of some sort.

Definition CoroSplit.cpp:876

void splitCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI) override

Definition CoroSplit.cpp:1993

static Function * createClone(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, CloneKind FKind, TargetTransformInfo &TTI)

Create a clone for a switch lowering.

void create() override

Clone the body of the original function into a resume function of some sort.

Definition CoroSplit.cpp:1090

const ParentTy * getParent() const

self_iterator getIterator()

NodeTy * getNextNode()

Get the next node, or nullptr for the list tail.

A range adaptor for a pair of iterators.

This class implements an extremely fast bulk output stream that can only output to a stream.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

constexpr char Args[]

Key for Kernel::Metadata::mArgs.

@ Fast

Attempts to make calls as fast as possible (e.g.

@ C

The default llvm calling convention, compatible with C.

@ Async

The "async continuation" lowering, where each suspend point creates a single continuation function.

@ RetconOnce

The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...

@ Retcon

The "returned-continuation" lowering, where each suspend point creates a single continuation function...

@ Switch

The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...

void suppressCoroAllocs(CoroIdInst *CoroId)

Replaces all @llvm.coro.alloc intrinsics calls associated with a given call @llvm....

void normalizeCoroutine(Function &F, coro::Shape &Shape, TargetTransformInfo &TTI)

CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, TargetTransformInfo &TTI, ArrayRef< Value * > Arguments, IRBuilder<> &)

Definition CoroSplit.cpp:1698

void replaceCoroFree(CoroIdInst *CoroId, bool Elide)

LLVM_ABI bool isTriviallyMaterializable(Instruction &I)

@ SwitchCleanup

The shared cleanup function for a switch lowering.

@ Continuation

An individual continuation function.

void salvageDebugInfo(SmallDenseMap< Argument *, AllocaInst *, 4 > &ArgToAllocaMap, DbgVariableRecord &DVR, bool UseEntryValue)

Attempts to rewrite the location operand of debug records in terms of the coroutine frame pointer,...

DiagnosticInfoOptimizationBase::Argument NV

This is an optimization pass for GlobalISel generic memory operations.

FunctionAddr VTableAddr Value

auto cast_if_present(const Y &Val)

cast_if_present - Functionally identical to cast, except that a null value is accepted.

UnaryFunction for_each(R &&Range, UnaryFunction F)

Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr, OptimizationRemarkEmitter *ORE=nullptr)

This function inlines the called function into the basic block of the caller.

detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)

zip iterator that assumes that all iteratees have the same length.

auto enumerate(FirstRange &&First, RestRanges &&...Rest)

Given two or more input ranges, returns a new range whose values are tuples (A, B,...

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

LLVM_ABI bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)

Check a function for errors, useful for use when debugging a pass.

LLVM_ABI LazyCallGraph::SCC & updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)

Helper to update the call graph after running a function pass.

LLVM_ABI LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)

Helper to update the call graph after running a CGSCC pass.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

bool isa_and_nonnull(const Y &Val)

AnalysisManager< LazyCallGraph::SCC, LazyCallGraph & > CGSCCAnalysisManager

The CGSCC analysis manager.

auto dyn_cast_or_null(const Y &Val)

LLVM_ABI BasicBlock::iterator skipDebugIntrinsics(BasicBlock::iterator It)

Advance It while it points to a debug instruction and return the result.

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)

iterator_range< SplittingIterator > split(StringRef Str, StringRef Separator)

Split the specified string over a separator and return a range-compatible iterable over its partition...

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

LLVM_ABI unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)

Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...

LLVM_ABI raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >

DWARFExpression::Operation Op

ArrayRef(const T &OneElt) -> ArrayRef< T >

ValueMap< const Value *, WeakTrackingVH > ValueToValueMapTy

LLVM_ABI void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)

Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

auto predecessors(const MachineBasicBlock *BB)

AnalysisManager< Function > FunctionAnalysisManager

Convenience typedef for the Function analysis manager.

static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)

Filter the DbgRecord range to DbgVariableRecord types only and downcast.

LLVM_ABI bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)

Remove all blocks that can not be reached from the function's entry.

LLVM_ABI bool isPotentiallyReachable(const Instruction *From, const Instruction *To, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet=nullptr, const DominatorTree *DT=nullptr, const LoopInfo *LI=nullptr)

Determine whether instruction 'To' is reachable from 'From', without passing through any blocks in Ex...

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.

This struct is a compact representation of a valid (non-zero power of two) alignment.

constexpr uint64_t value() const

This is a hole in the type system and should not be abused.

Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...

SmallPriorityWorklist< LazyCallGraph::SCC *, 1 > & CWorklist

Worklist of the SCCs queued for processing.

LLVM_ABI PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)

Definition CoroSplit.cpp:2212

LLVM_ABI CoroSplitPass(bool OptimizeFrame=false)

Definition CoroSplit.cpp:2168

BaseABITy CreateAndInitABI

CallInst * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)

GlobalVariable * AsyncFuncPointer

bool IsFrameInlineInStorage

SwitchInst * ResumeSwitch

BasicBlock * ResumeEntryBlock

SmallVector< CallInst *, 2 > SymmetricTransfers

SmallVector< CoroAwaitSuspendInst *, 4 > CoroAwaitSuspends

AsyncLoweringStorage AsyncLowering

FunctionType * getResumeFunctionType() const

IntegerType * getIndexType() const

CoroIdInst * getSwitchCoroId() const

SmallVector< CoroSizeInst *, 2 > CoroSizes

SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends

ConstantInt * getIndex(uint64_t Value) const

SwitchLoweringStorage SwitchLowering

CoroBeginInst * CoroBegin

BasicBlock::iterator getInsertPtAfterFramePtr() const

SmallVector< CoroIsInRampInst *, 2 > CoroIsInRampInsts

LLVM_ABI void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const

Deallocate memory according to the rules of the active lowering.

RetconLoweringStorage RetconLowering

SmallVector< CoroAlignInst *, 2 > CoroAligns

SmallVector< AnyCoroEndInst *, 4 > CoroEnds

SmallVector< CallInst *, 2 > SwiftErrorOps

unsigned getSwitchIndexField() const