LLVM: lib/Transforms/Coroutines/CoroFrame.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
40#include
41#include
42
43using namespace llvm;
44
45#define DEBUG_TYPE "coro-frame"
46
47namespace {
48class FrameTypeBuilder;
49
50struct FrameDataInfo {
51
52
54
55
57
60 : Spills(Spills), Allocas(Allocas) {}
61
64 for (const auto &P : Spills)
66 for (const auto &A : Allocas)
68 return Defs;
69 }
70
72 auto Itr = FieldIndexMap.find(V);
73 assert(Itr != FieldIndexMap.end() &&
74 "Value does not have a frame field index");
75 return Itr->second;
76 }
77
79 assert((LayoutIndexUpdateStarted || FieldIndexMap.count(V) == 0) &&
80 "Cannot set the index for the same field twice.");
81 FieldIndexMap[V] = Index;
82 }
83
85 auto Iter = FieldAlignMap.find(V);
86 assert(Iter != FieldAlignMap.end());
87 return Iter->second;
88 }
89
91 assert(FieldAlignMap.count(V) == 0);
92 FieldAlignMap.insert({V, AL});
93 }
94
96 auto Iter = FieldDynamicAlignMap.find(V);
97 assert(Iter != FieldDynamicAlignMap.end());
98 return Iter->second;
99 }
100
102 assert(FieldDynamicAlignMap.count(V) == 0);
103 FieldDynamicAlignMap.insert({V, Align});
104 }
105
107 auto Iter = FieldOffsetMap.find(V);
108 assert(Iter != FieldOffsetMap.end());
109 return Iter->second;
110 }
111
113 assert(FieldOffsetMap.count(V) == 0);
114 FieldOffsetMap.insert({V, Offset});
115 }
116
117
118 void updateLayoutIndex(FrameTypeBuilder &B);
119
120private:
121
122
123 bool LayoutIndexUpdateStarted = false;
124
125
126
128
129
132
133
135};
136}
137
138#ifndef NDEBUG
140 dbgs() << "------------- " << Title << " --------------\n";
141 for (const auto &E : Spills) {
142 E.first->dump();
143 dbgs() << " user: ";
145 I->dump();
146 }
147}
148
150 dbgs() << "------------- Allocas --------------\n";
151 for (const auto &A : Allocas) {
152 A.Alloca->dump();
153 }
154}
155#endif
156
157namespace {
158using FieldIDType = size_t;
159
160
161
162
163class FrameTypeBuilder {
164private:
165 struct Field {
166 uint64_t Size;
169 FieldIDType LayoutFieldIndex;
171 Align TyAlignment;
172 uint64_t DynamicAlignBuffer;
173 };
174
175 const DataLayout &DL;
176 LLVMContext &Context;
177 uint64_t StructSize = 0;
178 Align StructAlign;
179 bool IsFinished = false;
180
181 std::optional MaxFrameAlignment;
182
184 DenseMap<Value*, unsigned> FieldIndexByKey;
185
186public:
187 FrameTypeBuilder(LLVMContext &Context, const DataLayout &DL,
188 std::optional MaxFrameAlignment)
189 : DL(DL), Context(Context), MaxFrameAlignment(MaxFrameAlignment) {}
190
191
192
193 [[nodiscard]] FieldIDType addFieldForAlloca(AllocaInst *AI,
194 bool IsHeader = false) {
196
197
200 Ty = ArrayType::get(Ty, CI->getValue().getZExtValue());
201 else
202 report_fatal_error("Coroutines cannot handle non static allocas yet");
203 }
204
205 return addField(Ty, AI->getAlign(), IsHeader);
206 }
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235 void addFieldForAllocas(const Function &F, FrameDataInfo &FrameData,
236 coro::Shape &Shape, bool OptimizeFrame);
237
238
239 [[nodiscard]] FieldIDType addField(Type *Ty, MaybeAlign MaybeFieldAlignment,
240 bool IsHeader = false,
241 bool IsSpillOfValue = false) {
242 assert(!IsFinished && "adding fields to a finished builder");
243 assert(Ty && "must provide a type for a field");
244
245
246 uint64_t FieldSize = DL.getTypeAllocSize(Ty);
247
248
249
250 if (FieldSize == 0) {
251 return 0;
252 }
253
254
255
256
257
258 Align ABIAlign = DL.getABITypeAlign(Ty);
259 Align TyAlignment = ABIAlign;
260 if (IsSpillOfValue && MaxFrameAlignment && *MaxFrameAlignment < ABIAlign)
261 TyAlignment = *MaxFrameAlignment;
262 Align FieldAlignment = MaybeFieldAlignment.value_or(TyAlignment);
263
264
265
266
267 uint64_t DynamicAlignBuffer = 0;
268 if (MaxFrameAlignment && (FieldAlignment > *MaxFrameAlignment)) {
269 DynamicAlignBuffer =
271 FieldAlignment = *MaxFrameAlignment;
272 FieldSize = FieldSize + DynamicAlignBuffer;
273 }
274
275
277 if (IsHeader) {
279 StructSize = Offset + FieldSize;
280
281
282 } else {
284 }
285
286 Fields.push_back({FieldSize, Offset, Ty, 0, FieldAlignment, TyAlignment,
287 DynamicAlignBuffer});
288 return Fields.size() - 1;
289 }
290
291
292 StructType *finish(StringRef Name);
293
294 uint64_t getStructSize() const {
295 assert(IsFinished && "not yet finished!");
296 return StructSize;
297 }
298
299 Align getStructAlign() const {
300 assert(IsFinished && "not yet finished!");
301 return StructAlign;
302 }
303
304 FieldIDType getLayoutFieldIndex(FieldIDType Id) const {
305 assert(IsFinished && "not yet finished!");
306 return Fields[Id].LayoutFieldIndex;
307 }
308
309 Field getLayoutField(FieldIDType Id) const {
310 assert(IsFinished && "not yet finished!");
311 return Fields[Id];
312 }
313};
314}
315
316void FrameDataInfo::updateLayoutIndex(FrameTypeBuilder &B) {
317 auto Updater = [&](Value *I) {
318 auto Field = B.getLayoutField(getFieldIndex(I));
319 setFieldIndex(I, Field.LayoutFieldIndex);
321 uint64_t dynamicAlign =
322 Field.DynamicAlignBuffer
324 : 0;
325 setDynamicAlign(I, dynamicAlign);
327 };
328 LayoutIndexUpdateStarted = true;
329 for (auto &S : Spills)
330 Updater(S.first);
331 for (const auto &A : Allocas)
332 Updater(A.Alloca);
333 LayoutIndexUpdateStarted = false;
334}
335
336void FrameTypeBuilder::addFieldForAllocas(const Function &F,
337 FrameDataInfo &FrameData,
338 coro::Shape &Shape,
339 bool OptimizeFrame) {
342
343
345 for (auto AllocaList : NonOverlapedAllocas) {
346 auto *LargestAI = *AllocaList.begin();
347 FieldIDType Id = addFieldForAlloca(LargestAI);
348 for (auto *Alloca : AllocaList)
349 FrameData.setFieldIndex(Alloca, Id);
350 }
351 });
352
353 if (!OptimizeFrame) {
354 for (const auto &A : FrameData.Allocas) {
355 AllocaInst *Alloca = A.Alloca;
356 NonOverlapedAllocas.emplace_back(AllocaSetType(1, Alloca));
357 }
358 return;
359 }
360
361
362
363
364
365
366
367
368
369
370
371
372
373 DenseMap<SwitchInst *, BasicBlock *> DefaultSuspendDest;
374 for (auto *CoroSuspendInst : Shape.CoroSuspends) {
375 for (auto *U : CoroSuspendInst->users()) {
377 auto *SWI = const_cast<SwitchInst *>(ConstSWI);
378 DefaultSuspendDest[SWI] = SWI->getDefaultDest();
379 SWI->setDefaultDest(SWI->getSuccessor(1));
380 }
381 }
382 }
383
384 auto ExtractAllocas = [&]() {
385 AllocaSetType Allocas;
386 Allocas.reserve(FrameData.Allocas.size());
387 for (const auto &A : FrameData.Allocas)
388 Allocas.push_back(A.Alloca);
389 return Allocas;
390 };
391 StackLifetime StackLifetimeAnalyzer(F, ExtractAllocas(),
392 StackLifetime::LivenessType::May);
393 StackLifetimeAnalyzer.run();
394 auto DoAllocasInterfere = [&](const AllocaInst *AI1, const AllocaInst *AI2) {
395 return StackLifetimeAnalyzer.getLiveRange(AI1).overlaps(
396 StackLifetimeAnalyzer.getLiveRange(AI2));
397 };
398 auto GetAllocaSize = [&](const coro::AllocaInfo &A) {
399 std::optional RetSize = A.Alloca->getAllocationSize(DL);
400 assert(RetSize && "Variable Length Arrays (VLA) are not supported.\n");
401 assert(!RetSize->isScalable() && "Scalable vectors are not yet supported");
402 return RetSize->getFixedValue();
403 };
404
405
406
407
408 sort(FrameData.Allocas, [&](const auto &Iter1, const auto &Iter2) {
409 return GetAllocaSize(Iter1) > GetAllocaSize(Iter2);
410 });
411 for (const auto &A : FrameData.Allocas) {
412 AllocaInst *Alloca = A.Alloca;
413 bool Merged = false;
414
415
416
417 for (auto &AllocaSet : NonOverlapedAllocas) {
418 assert(!AllocaSet.empty() && "Processing Alloca Set is not empty.\n");
419 bool NoInterference = none_of(AllocaSet, [&](auto Iter) {
420 return DoAllocasInterfere(Alloca, Iter);
421 });
422
423
424
425
426
427
428 bool Alignable = [&]() -> bool {
429 auto *LargestAlloca = *AllocaSet.begin();
430 return LargestAlloca->getAlign().value() % Alloca->getAlign().value() ==
431 0;
432 }();
433 bool CouldMerge = NoInterference && Alignable;
434 if (!CouldMerge)
435 continue;
436 AllocaSet.push_back(Alloca);
437 Merged = true;
438 break;
439 }
440 if (!Merged) {
441 NonOverlapedAllocas.emplace_back(AllocaSetType(1, Alloca));
442 }
443 }
444
445
446 for (auto SwitchAndDefaultDest : DefaultSuspendDest) {
447 SwitchInst *SWI = SwitchAndDefaultDest.first;
448 BasicBlock *DestBB = SwitchAndDefaultDest.second;
450 }
451
453 : NonOverlapedAllocas) {
454 if (AllocaSet.size() > 1) {
455 dbgs() << "In Function:" << F.getName() << "\n";
456 dbgs() << "Find Union Set "
457 << "\n";
458 dbgs() << "\tAllocas are \n";
459 for (auto Alloca : AllocaSet)
460 dbgs() << "\t\t" << *Alloca << "\n";
461 }
462 });
463}
464
465StructType *FrameTypeBuilder::finish(StringRef Name) {
466 assert(!IsFinished && "already finished!");
467
468
469
472 for (auto &Field : Fields) {
475 }
476
477
479 StructSize = SizeAndAlign.first;
480 StructAlign = SizeAndAlign.second;
481
482 auto getField = [](const OptimizedStructLayoutField &LayoutField) -> Field & {
483 return *static_cast<Field *>(const_cast<void*>(LayoutField.Id));
484 };
485
486
487
489 for (auto &LayoutField : LayoutFields) {
490 auto &F = getField(LayoutField);
491 if ((F.TyAlignment, LayoutField.Offset))
492 return true;
493 }
494 return false;
495 }();
496
497
499 FieldTypes.reserve(LayoutFields.size() * 3 / 2);
500 uint64_t LastOffset = 0;
501 for (auto &LayoutField : LayoutFields) {
502 auto &F = getField(LayoutField);
503
504 auto Offset = LayoutField.Offset;
505
506
507
508
510 if (Offset != LastOffset) {
511 if (Packed || alignTo(LastOffset, F.TyAlignment) != Offset)
512 FieldTypes.push_back(ArrayType::get(Type::getInt8Ty(Context),
513 Offset - LastOffset));
514 }
515
517 F.LayoutFieldIndex = FieldTypes.size();
518
520 if (F.DynamicAlignBuffer) {
522 ArrayType::get(Type::getInt8Ty(Context), F.DynamicAlignBuffer));
523 }
524 LastOffset = Offset + F.Size;
525 }
526
528
529#ifndef NDEBUG
530
531 auto Layout = DL.getStructLayout(Ty);
532 for (auto &F : Fields) {
534 assert(Layout->getElementOffset(F.LayoutFieldIndex) == F.Offset);
535 }
536#endif
537
538 IsFinished = true;
539
540 return Ty;
541}
542
545 for (auto *V : FrameData.getAllDefs()) {
547 continue;
548
549 auto CacheIt = [&DIVarCache, V](const auto &Container) {
551 return DDI->getExpression()->getNumElements() == 0;
552 });
553 if (I != Container.end())
554 DIVarCache.insert({V, (*I)->getVariable()});
555 };
558 }
559}
560
561
562
564 if (Ty->isIntegerTy()) {
565
570 return MDName->getString();
571 }
572
573 if (Ty->isFloatingPointTy()) {
574 if (Ty->isFloatTy())
575 return "__float_";
576 if (Ty->isDoubleTy())
577 return "__double_";
578 return "__floating_type_";
579 }
580
581 if (Ty->isPointerTy())
582 return "PointerType";
583
584 if (Ty->isStructTy()) {
586 return "__LiteralStructType_";
587
588 auto Name = Ty->getStructName();
589
591 for (auto &Iter : Buffer)
592 if (Iter == '.' || Iter == ':')
593 Iter = '_';
594 auto *MDName = MDString::get(Ty->getContext(), Buffer.str());
595 return MDName->getString();
596 }
597
598 return "UnknownType";
599}
600
603 unsigned LineNum,
606 return DT;
607
609
610 DIType *RetType = nullptr;
611
612 if (Ty->isIntegerTy()) {
614 RetType = Builder.createBasicType(Name, BitWidth, dwarf::DW_ATE_signed,
615 llvm::DINode::FlagArtificial);
616 } else if (Ty->isFloatingPointTy()) {
617 RetType = Builder.createBasicType(Name, Layout.getTypeSizeInBits(Ty),
618 dwarf::DW_ATE_float,
619 llvm::DINode::FlagArtificial);
620 } else if (Ty->isPointerTy()) {
621
622
623
624
625
626
627
628 RetType =
631 std::nullopt, Name);
632 } else if (Ty->isStructTy()) {
633 auto *DIStruct = Builder.createStructType(
634 Scope, Name, Scope->getFile(), LineNum, Layout.getTypeSizeInBits(Ty),
636 llvm::DINode::FlagArtificial, nullptr, llvm::DINodeArray());
637
640 for (unsigned I = 0; I < StructTy->getNumElements(); I++) {
641 DIType *DITy = solveDIType(Builder, StructTy->getElementType(I), Layout,
642 DIStruct, LineNum, DITypeCache);
644 Elements.push_back(Builder.createMemberType(
645 DIStruct, DITy->getName(), DIStruct->getFile(), LineNum,
648 llvm::DINode::FlagArtificial, DITy));
649 }
650
651 Builder.replaceArrays(DIStruct, Builder.getOrCreateArray(Elements));
652
653 RetType = DIStruct;
654 } else {
655 LLVM_DEBUG(dbgs() << "Unresolved Type: " << *Ty << "\n");
657 auto *CharSizeType = Builder.createBasicType(
658 Name, 8, dwarf::DW_ATE_unsigned_char, llvm::DINode::FlagArtificial);
659
660 if (Size <= 8)
661 RetType = CharSizeType;
662 else {
663 if (Size % 8 != 0)
665
666 RetType = Builder.createArrayType(
668 Builder.getOrCreateArray(Builder.getOrCreateSubrange(0, Size / 8)));
669 }
670 }
671
672 DITypeCache.insert({Ty, RetType});
673 return RetType;
674}
675
676
677
678
679
680
681
682
683
684
685
686
687
689 FrameDataInfo &FrameData) {
691
692
693
694 if (!DIS || !DIS->getUnit())
695 return;
696
698 DIS->getUnit()->getSourceLanguage().getUnversionedName())) ||
699 DIS->getUnit()->getEmissionKind() !=
701 return;
702
704 "We could only build debug infomation for C++ coroutine now.\n");
705
706 DIBuilder DBuilder(*F.getParent(), false);
707
709 unsigned LineNum = DIS->getLine();
710
712 DIS->getUnit(), Twine(F.getName() + ".coro_frame_ty").str(),
713 DFile, LineNum, Shape.FrameSize * 8,
714 Shape.FrameAlign.value() * 8, llvm::DINode::FlagArtificial, nullptr,
715 llvm::DINodeArray());
719
722
726
728 NameCache.insert({ResumeIndex, "__resume_fn"});
729 NameCache.insert({DestroyIndex, "__destroy_fn"});
730 NameCache.insert({IndexIndex, "__coro_index"});
731
733 *DestroyFnTy = FrameTy->getElementType(DestroyIndex),
735
743
744
745
747 "__coro_index",
749 ? 8
751 dwarf::DW_ATE_unsigned_char)});
752
753 for (auto *V : FrameData.getAllDefs()) {
754 auto It = DIVarCache.find(V);
755 if (It == DIVarCache.end())
756 continue;
757
758 auto Index = FrameData.getFieldIndex(V);
759
760 NameCache.insert({Index, It->second->getName()});
761 TyCache.insert({Index, It->second->getType()});
762 }
763
764
766
767 OffsetCache.insert({ResumeIndex, {8, 0}});
768 OffsetCache.insert({DestroyIndex, {8, 8}});
770 {IndexIndex,
772
773 for (auto *V : FrameData.getAllDefs()) {
774 auto Index = FrameData.getFieldIndex(V);
775
777 {Index, {FrameData.getAlign(V).value(), FrameData.getOffset(V)}});
778 }
779
781
782
783
784
785 unsigned UnknownTypeNum = 0;
786 for (unsigned Index = 0; Index < FrameTy->getNumElements(); Index++) {
787 auto OCIt = OffsetCache.find(Index);
788 if (OCIt == OffsetCache.end())
789 continue;
790
791 std::string Name;
795 DIType *DITy = nullptr;
796
798 assert(Ty->isSized() && "We can't handle type which is not sized.\n");
800 AlignInBits = OCIt->second.first * 8;
801 OffsetInBits = OCIt->second.second * 8;
802
803 if (auto It = NameCache.find(Index); It != NameCache.end()) {
804 Name = It->second.str();
805 DITy = TyCache[Index];
806 } else {
807 DITy = solveDIType(DBuilder, Ty, Layout, FrameDITy, LineNum, DITypeCache);
808 assert(DITy && "SolveDIType shouldn't return nullptr.\n");
810 Name += "_" + std::to_string(UnknownTypeNum);
811 UnknownTypeNum++;
812 }
813
815 FrameDITy, Name, DFile, LineNum, SizeInBits, AlignInBits, OffsetInBits,
816 llvm::DINode::FlagArtificial, DITy));
817 }
818
820
821 auto *FrameDIVar =
823 FrameDITy, true, DINode::FlagArtificial);
824
825
826
827
828
829
830
831
832 auto RetainedNodes = DIS->getRetainedNodes();
834 RetainedNodes.end());
835 RetainedNodesVec.push_back(FrameDIVar);
837
838
839
842 assert(FrameDIVar->isValidLocationForIntrinsic(DILoc));
843
849 It->getParent()->insertDbgRecordBefore(NewDVR, It);
850}
851
852
853
854
855
856
857
858
859
861 FrameDataInfo &FrameData,
862 bool OptimizeFrame) {
865
866
867 std::optional MaxFrameAlignment;
870 FrameTypeBuilder B(C, DL, MaxFrameAlignment);
871
873 std::optional SwitchIndexFieldId;
874
877
878
879
880 (void)B.addField(FnPtrTy, std::nullopt, true);
881 (void)B.addField(FnPtrTy, std::nullopt, true);
882
883
884
885
886 if (PromiseAlloca)
887 FrameData.setFieldIndex(
888 PromiseAlloca, B.addFieldForAlloca(PromiseAlloca, true));
889
890
891
894
895 SwitchIndexFieldId = B.addField(IndexType, std::nullopt);
896 } else {
897 assert(PromiseAlloca == nullptr && "lowering doesn't support promises");
898 }
899
900
901
902 B.addFieldForAllocas(F, FrameData, Shape, OptimizeFrame);
903
904
905
906
908
909
910 FrameData.Allocas.emplace_back(
912
913 for (auto &S : FrameData.Spills) {
914 Type *FieldType = S.first->getType();
916
917
919 if (A->hasByValAttr()) {
920 FieldType = A->getParamByValType();
921 MA = A->getParamAlign();
922 }
923 }
924 FieldIDType Id =
925 B.addField(FieldType, MA, false , true );
926 FrameData.setFieldIndex(S.first, Id);
927 }
928
931 Name.append(".Frame");
932 return B.finish(Name);
933 }();
934
935 FrameData.updateLayoutIndex(B);
938
939 switch (Shape.ABI) {
941
942 auto IndexField = B.getLayoutField(*SwitchIndexFieldId);
946
947
948
950 break;
951 }
952
953
958 = (B.getStructSize() <= Id->getStorageSize() &&
959 B.getStructAlign() <= Id->getStorageAlignment());
960 break;
961 }
965
966
972 "The alignment requirment of frame variables cannot be higher than "
973 "the alignment of the async function context");
974 }
975 break;
976 }
977 }
978
979 return FrameTy;
980}
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1011
1012
1013
1014
1015 auto GetFramePointer = [&](Value *Orig) -> Value * {
1016 FieldIDType Index = FrameData.getFieldIndex(Orig);
1020 };
1021
1024 auto Count = CI->getValue().getZExtValue();
1025 if (Count > 1) {
1027 }
1028 } else {
1029 report_fatal_error("Coroutines cannot handle non static allocas yet");
1030 }
1031 }
1032
1034 Builder.CreateInBoundsGEP(FrameTy, FramePtr, Indices));
1036 if (FrameData.getDynamicAlign(Orig) != 0) {
1040 auto *PtrValue = Builder.CreatePtrToInt(GEP, IntPtrTy);
1041 auto *AlignMask =
1042 ConstantInt::get(IntPtrTy, AI->getAlign().value() - 1);
1043 PtrValue = Builder.CreateAdd(PtrValue, AlignMask);
1044 PtrValue = Builder.CreateAnd(PtrValue, Builder.CreateNot(AlignMask));
1045 return Builder.CreateIntToPtr(PtrValue, AI->getType());
1046 }
1047
1048
1049
1050
1051
1052
1053
1054 if (GEP->getType() != Orig->getType())
1055 return Builder.CreateAddrSpaceCast(GEP, Orig->getType(),
1056 Orig->getName() + Twine(".cast"));
1057 }
1058 return GEP;
1059 };
1060
1061 for (auto const &E : FrameData.Spills) {
1063 auto SpillAlignment = Align(FrameData.getAlign(Def));
1064
1065
1067
1068 Type *ByValTy = nullptr;
1070
1071
1072 Arg->getParent()->removeParamAttr(Arg->getArgNo(), Attribute::Captures);
1073
1074 if (Arg->hasByValAttr())
1075 ByValTy = Arg->getParamByValType();
1076 }
1077
1078 auto Index = FrameData.getFieldIndex(Def);
1079 Builder.SetInsertPoint(InsertPt->getParent(), InsertPt);
1080 auto *G = Builder.CreateConstInBoundsGEP2_32(
1081 FrameTy, FramePtr, 0, Index, Def->getName() + Twine(".spill.addr"));
1082 if (ByValTy) {
1083
1084
1085 auto *Value = Builder.CreateLoad(ByValTy, Def);
1086 Builder.CreateAlignedStore(Value, G, SpillAlignment);
1087 } else {
1088 Builder.CreateAlignedStore(Def, G, SpillAlignment);
1089 }
1090
1092 Value *CurrentReload = nullptr;
1093 for (auto *U : E.second) {
1094
1095
1096
1097 if (CurrentBlock != U->getParent()) {
1099 Builder.SetInsertPoint(CurrentBlock,
1101
1102 auto *GEP = GetFramePointer(E.first);
1103 GEP->setName(E.first->getName() + Twine(".reload.addr"));
1104 if (ByValTy)
1105 CurrentReload = GEP;
1106 else
1107 CurrentReload = Builder.CreateAlignedLoad(
1109 SpillAlignment, E.first->getName() + Twine(".reload"));
1110
1112
1113
1114
1115 if (F->getSubprogram()) {
1116 auto *CurDef = Def;
1119
1120 if (LdInst->getPointerOperandType() != LdInst->getType())
1121 break;
1122 CurDef = LdInst->getPointerOperand();
1124 break;
1126 }
1127 }
1128
1130
1131
1132
1135 DDI->getExpression(), DDI->getDebugLoc(),
1137 Builder.GetInsertPoint()->getParent()->insertDbgRecordBefore(
1138 NewDVR, Builder.GetInsertPoint());
1139
1140
1142 };
1144 }
1145
1148
1149
1150
1151 if (F->getSubprogram()) {
1152 auto *CurDef = Def;
1155
1156 if (LdInst->getPointerOperandType() != LdInst->getType())
1157 break;
1158 CurDef = LdInst->getPointerOperand();
1160 break;
1162 }
1163 }
1164
1165 auto SalvageOneCoro = [&](auto *DDI) {
1166
1167
1168
1169
1170
1172 Type *Ty = VAM->getValue()->getType();
1173
1176 DDI->getExpression(), DDI->getDebugLoc(),
1179 Builder.GetInsertPoint()->getParent()->insertDbgRecordBefore(
1180 NewDVR, Builder.GetInsertPoint());
1181
1182
1184 };
1185 for_each(DVRDeclareValues, SalvageOneCoro);
1186
1187
1188
1189
1191 assert(PN->getNumIncomingValues() == 1 &&
1192 "unexpected number of incoming "
1193 "values in the PHINode");
1194 PN->replaceAllUsesWith(CurrentReload);
1195 PN->eraseFromParent();
1196 continue;
1197 }
1198
1199
1200
1201 U->replaceUsesOfWith(Def, CurrentReload);
1202
1203
1205 DVR.replaceVariableLocationOp(Def, CurrentReload, true);
1206 }
1207 }
1208
1210
1213 SpillBlock->splitBasicBlock(&SpillBlock->front(), "PostSpill");
1215
1216
1219
1220 Builder.SetInsertPoint(SpillBlock, SpillBlock->begin());
1221 for (const auto &P : FrameData.Allocas) {
1223 auto *G = GetFramePointer(Alloca);
1224
1225
1228 if (I->isLifetimeStartOrEnd())
1229 I->eraseFromParent();
1230 }
1231
1232
1233
1234 G->takeName(Alloca);
1237 }
1238 return;
1239 }
1240
1241
1242
1243
1244
1245
1249 for (const auto &A : FrameData.Allocas) {
1251 UsersToUpdate.clear();
1254
1255
1256
1257 if (I->isLifetimeStartOrEnd())
1258 I->eraseFromParent();
1261 }
1262
1263 if (UsersToUpdate.empty())
1264 continue;
1265 auto *G = GetFramePointer(Alloca);
1266 G->setName(Alloca->getName() + Twine(".reload.addr"));
1267
1270 for (auto *DVR : DbgVariableRecords)
1271 DVR->replaceVariableLocationOp(Alloca, G);
1272
1274 I->replaceUsesOfWith(Alloca, G);
1275 }
1277 for (const auto &A : FrameData.Allocas) {
1279 if (A.MayWriteBeforeCoroBegin) {
1280
1283 "Coroutines cannot handle copying of array allocas yet");
1284
1285 auto *G = GetFramePointer(Alloca);
1287 Builder.CreateStore(Value, G);
1288 }
1289
1290
1291
1292 for (const auto &Alias : A.Aliases) {
1293 auto *FramePtr = GetFramePointer(Alloca);
1294 auto &Value = *Alias.second;
1296 auto *AliasPtr =
1297 Builder.CreatePtrAdd(FramePtr, ConstantInt::get(ITy, Value));
1298 Alias.first->replaceUsesWithIf(
1300 }
1301 }
1302
1303
1304
1305
1306
1309
1310 bool HasAccessingPromiseBeforeCB = llvm::any_of(PA->uses(), [&](Use &U) {
1311 auto *Inst = dyn_cast(U.getUser());
1312 if (!Inst || DT.dominates(Shape.CoroBegin, Inst))
1313 return false;
1314
1315 if (auto *CI = dyn_cast(Inst)) {
1316
1317
1318
1319
1320 if (CI->onlyReadsMemory() ||
1321 CI->onlyReadsMemory(CI->getArgOperandNo(&U)))
1322 return false;
1323 return true;
1324 }
1325
1327
1328
1330
1331
1332
1333
1335 });
1336 if (HasAccessingPromiseBeforeCB) {
1338 auto *G = GetFramePointer(PA);
1339 auto *Value = Builder.CreateLoad(PA->getAllocatedType(), PA);
1340 Builder.CreateStore(Value, G);
1341 }
1342 }
1343}
1344
1345
1346
1350 PHINode *UntilPHI = nullptr) {
1352 do {
1353 int Index = PN->getBasicBlockIndex(InsertedBB);
1354 Value *V = PN->getIncomingValue(Index);
1356 V->getType(), 1, V->getName() + Twine(".") + SuccBB->getName());
1359 PN->setIncomingValue(Index, InputV);
1361 } while (PN != UntilPHI);
1362}
1363
1364
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1398 Builder.CreateUnreachable();
1399
1400
1401 auto *NewCleanupPadBB =
1403 CleanupPadBB->getName() + Twine(".corodispatch"),
1404 CleanupPadBB->getParent(), CleanupPadBB);
1405 Builder.SetInsertPoint(NewCleanupPadBB);
1406 auto *SwitchType = Builder.getInt8Ty();
1407 auto *SetDispatchValuePN =
1408 Builder.CreatePHI(SwitchType, pred_size(CleanupPadBB));
1410 CleanupPad->insertAfter(SetDispatchValuePN->getIterator());
1411 auto *SwitchOnDispatch = Builder.CreateSwitch(SetDispatchValuePN, UnreachBB,
1413
1414 int SwitchIndex = 0;
1417
1419 CleanupPadBB->getName() +
1420 Twine(".from.") + Pred->getName(),
1421 CleanupPadBB->getParent(), CleanupPadBB);
1423 CaseBB->setName(CleanupPadBB->getName() + Twine(".from.") +
1424 Pred->getName());
1425 Builder.SetInsertPoint(CaseBB);
1426 Builder.CreateBr(CleanupPadBB);
1428
1429
1430 setUnwindEdgeTo(Pred->getTerminator(), NewCleanupPadBB);
1431
1432
1433 auto *SwitchConstant = ConstantInt::get(SwitchType, SwitchIndex);
1434 SetDispatchValuePN->addIncoming(SwitchConstant, Pred);
1435 SwitchOnDispatch->addCase(SwitchConstant, CaseBB);
1436 SwitchIndex++;
1437 }
1438}
1439
1442 for (auto &BB : F) {
1443 for (auto &Phi : BB.phis()) {
1444 if (Phi.getNumIncomingValues() == 1) {
1446 } else
1447 break;
1448 }
1449 }
1450 while (!Worklist.empty()) {
1452 auto *OriginalValue = Phi->getIncomingValue(0);
1453 Phi->replaceAllUsesWith(OriginalValue);
1454 }
1455}
1456
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481 if (!BB.empty()) {
1482 if (auto *CleanupPad =
1488
1489
1490 assert(CS->getUnwindDest() == &BB);
1491 (void)CS;
1493 return;
1494 }
1495 }
1496 }
1497 }
1498
1500 PHINode *ReplPHI = nullptr;
1501 if (!BB.empty()) {
1502 if ((LandingPad =
1504
1505
1506
1509 ReplPHI->takeName(LandingPad);
1511
1512
1513 }
1514 }
1515
1518 auto *IncomingBB = ehAwareSplitEdge(Pred, &BB, LandingPad, ReplPHI);
1519 IncomingBB->setName(BB.getName() + Twine(".from.") + Pred->getName());
1520
1521
1522
1524 }
1525
1526 if (LandingPad) {
1527
1528
1530 }
1531}
1532
1535
1538 if (PN->getNumIncomingValues() > 1)
1540
1543}
1544
1545
1546
1548 auto *BB = I->getParent();
1549 if (&BB->front() == I) {
1550 if (BB->getSinglePredecessor()) {
1551 BB->setName(Name);
1552 return BB;
1553 }
1554 }
1555 return BB->splitBasicBlock(I, Name);
1556}
1557
1558
1559
1564
1565
1566
1568 unsigned depth = 3) {
1569
1570
1571 if (depth == 0) return false;
1572
1573
1575 return true;
1576
1577
1580 return false;
1581 }
1582
1583
1584 return true;
1585}
1586
1588
1589
1590
1591 for (auto *U : AI->users()) {
1593 if (!FI) continue;
1594
1596 return true;
1597 }
1598
1599
1600 return false;
1601}
1602
1603
1604
1607 for (auto *AI : LocalAllocas) {
1609
1610
1611
1612 Value *StackSave = nullptr;
1614 StackSave = Builder.CreateStackSave();
1615
1616
1617 auto Alloca = Builder.CreateAlloca(Builder.getInt8Ty(), AI->getSize());
1619
1620 for (auto *U : AI->users()) {
1621
1623 U->replaceAllUsesWith(Alloca);
1624
1625
1626
1627
1628 } else {
1630 if (StackSave) {
1631 Builder.SetInsertPoint(FI);
1632 Builder.CreateStackRestore(StackSave);
1633 }
1634 }
1636 }
1637
1639 }
1640}
1641
1642
1645
1648
1649 auto Call = Builder.CreateCall(FnTy, Fn, {});
1651
1652 return Call;
1653}
1654
1655
1656
1657
1660
1662 {V->getType()}, false);
1664
1665 auto Call = Builder.CreateCall(FnTy, Fn, { V });
1667
1668 return Call;
1669}
1670
1671
1672
1673
1674
1675
1681
1682
1683
1684 auto ValueBeforeCall = Builder.CreateLoad(ValueTy, Alloca);
1686
1687
1688
1689
1691 Builder.SetInsertPoint(Call->getNextNode());
1692 } else {
1694 Builder.SetInsertPoint(Invoke->getNormalDest()->getFirstNonPHIOrDbg());
1695 }
1696
1697
1699 Builder.CreateStore(ValueAfterCall, Alloca);
1700
1701 return Addr;
1702}
1703
1704
1705
1709
1710
1713 continue;
1714
1717
1719
1720
1722 }
1723
1724
1726}
1727
1728
1729
1730
1731
1736 F.getEntryBlock().getFirstNonPHIOrDbg());
1737
1740
1741
1742
1743
1744 auto Alloca = Builder.CreateAlloca(ValueTy, ArgTy->getAddressSpace());
1746
1747
1749 Builder.CreateStore(InitialValue, Alloca);
1750
1751
1754 }
1755
1756
1757 for (auto *End : Shape.CoroEnds) {
1758 Builder.SetInsertPoint(End);
1759 auto FinalValue = Builder.CreateLoad(ValueTy, Alloca);
1761 }
1762
1763
1764 AllocasToPromote.push_back(Alloca);
1766}
1767
1768
1769
1772
1773
1774 for (auto &Arg : F.args()) {
1775 if (!Arg.hasSwiftErrorAttr()) continue;
1776
1778 break;
1779 }
1780
1781
1782 for (auto &Inst : F.getEntryBlock()) {
1784 if (!Alloca || !Alloca->isSwiftError()) continue;
1785
1786
1788
1789 AllocasToPromote.push_back(Alloca);
1791 }
1792
1793
1794
1795 if (!AllocasToPromote.empty()) {
1798 }
1799}
1800
1801
1802
1803
1804
1808 if (F.hasOptNone())
1809 return;
1810
1811
1813 DomSet.insert(&F.getEntryBlock());
1818 "should have split coro.suspend into its own block");
1820 }
1821
1824 if (!AI)
1825 continue;
1826
1828 bool Valid = true;
1830
1833 return II->getIntrinsicID() == Intrinsic::lifetime_start;
1834 return false;
1835 };
1836
1840 return true;
1841 }
1842 if (!U->hasOneUse() || U->stripPointerCasts() != AI)
1843 return false;
1845 Lifetimes.push_back(U->user_back());
1846 return true;
1847 }
1848 return false;
1849 };
1850
1853
1854
1855
1856
1859
1860
1861 if (collectLifetimeStart(UI, AI))
1862 continue;
1863 Valid = false;
1864 break;
1865 }
1866 }
1867
1868
1869 if (Valid && Lifetimes.size() != 0) {
1870 auto *NewLifetime = Lifetimes[0]->clone();
1871 NewLifetime->replaceUsesOfWith(NewLifetime->getOperand(0), AI);
1872 NewLifetime->insertBefore(DomBB->getTerminator()->getIterator());
1873
1874
1876 S->eraseFromParent();
1877
1878 break;
1879 }
1880 }
1881 }
1882}
1883
1884static std::optional<std::pair<Value &, DIExpression &>>
1887 DIExpression *Expr, bool SkipOutermostLoad) {
1889 auto InsertPt = F->getEntryBlock().getFirstInsertionPt();
1891 ++InsertPt;
1892 Builder.SetInsertPoint(&F->getEntryBlock(), InsertPt);
1893
1896 Storage = LdInst->getPointerOperand();
1897
1898
1899
1900
1901
1902
1903 if (!SkipOutermostLoad)
1906 Storage = StInst->getValueOperand();
1907 } else {
1912 AdditionalValues);
1913 if ( || !AdditionalValues.empty()) {
1914
1915
1916 break;
1917 }
1918 Storage = Op;
1920 }
1921 SkipOutermostLoad = false;
1922 }
1923 if (!Storage)
1924 return std::nullopt;
1925
1927 const bool IsSwiftAsyncArg =
1928 StorageAsArg && StorageAsArg->hasAttribute(Attribute::SwiftAsync);
1929
1930
1931
1932
1933 if (IsSwiftAsyncArg && UseEntryValue && !Expr->isEntryValue() &&
1936
1937
1938
1939
1940
1941 if (StorageAsArg && !IsSwiftAsyncArg) {
1942 auto &Cached = ArgToAllocaMap[StorageAsArg];
1943 if (!Cached) {
1944 Cached = Builder.CreateAlloca(Storage->getType(), 0, nullptr,
1945 Storage->getName() + ".debug");
1946 Builder.CreateStore(Storage, Cached);
1947 }
1948 Storage = Cached;
1949
1950
1951
1952
1953
1954
1955
1957 }
1958
1960 return {{*Storage, *Expr}};
1961}
1962
1966
1968
1969
1972
1973 auto SalvagedInfo =
1976 if (!SalvagedInfo)
1977 return;
1978
1979 Value *Storage = &SalvagedInfo->first;
1981
1984
1985
1986
1989 std::optionalBasicBlock::iterator InsertPt;
1991 InsertPt = I->getInsertionPointAfterDef();
1992
1993 DebugLoc ILoc = I->getDebugLoc();
1995 if (ILoc && DVRLoc &&
1996 DVRLoc->getScope()->getSubprogram() ==
1997 ILoc->getScope()->getSubprogram())
2000 InsertPt = F->getEntryBlock().begin();
2001 if (InsertPt) {
2003
2004
2005
2009 Type *Ty = VAM->getValue()->getType();
2010 if (Ty->isPointerTy())
2012 else
2014 }
2015 }
2016 (*InsertPt)->getParent()->insertDbgRecordBefore(&DVR, *InsertPt);
2017 }
2018 }
2019}
2020
2023
2026
2030 }
2031
2032
2033
2034
2036 if (auto *Save = CSI->getCoroSave())
2039 }
2040
2041
2044
2045
2046
2047
2048
2050 auto *MustTailCallFn = AsyncEnd->getMustTailCallFunction();
2051 if (!MustTailCallFn)
2052 continue;
2057 AsyncEnd->getDebugLoc(), MustTailCallFn, TTI, Arguments, Builder);
2059 }
2060 }
2061
2062
2063
2065
2066
2067
2069}
2070
2074
2079
2080
2082
2084
2085
2090 LocalAllocas, F, Checker, DT, Shape);
2092
2095
2099
2100
2101 FrameDataInfo FrameData(Spills, Allocas);
2104
2106
2109
2110 for (auto *I : DeadInstructions)
2111 I->eraseFromParent();
2112}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Lower Kernel Arguments
Rewrite undef for false bool rewritePHIs(Function &F, UniformityInfo &UA, DominatorTree *DT)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static void cleanupSinglePredPHIs(Function &F)
Definition CoroFrame.cpp:1440
static void eliminateSwiftError(Function &F, coro::Shape &Shape)
Eliminate all problematic uses of swifterror arguments and allocas from the function.
Definition CoroFrame.cpp:1770
static void lowerLocalAllocas(ArrayRef< CoroAllocaAllocInst * > LocalAllocas, SmallVectorImpl< Instruction * > &DeadInsts)
Turn each of the given local allocas into a normal (dynamic) alloca instruction.
Definition CoroFrame.cpp:1605
static Value * emitSetSwiftErrorValue(IRBuilder<> &Builder, Value *V, coro::Shape &Shape)
Set the given value as the current swifterror value.
Definition CoroFrame.cpp:1658
static Value * emitSetAndGetSwiftErrorValueAround(Instruction *Call, AllocaInst *Alloca, coro::Shape &Shape)
Set the swifterror value from the given alloca before a call, then put in back in the alloca afterwar...
Definition CoroFrame.cpp:1676
static void cacheDIVar(FrameDataInfo &FrameData, DenseMap< Value *, DILocalVariable * > &DIVarCache)
Definition CoroFrame.cpp:543
static bool localAllocaNeedsStackSave(CoroAllocaAllocInst *AI)
Definition CoroFrame.cpp:1587
static void dumpAllocas(const SmallVectorImpl< coro::AllocaInfo > &Allocas)
Definition CoroFrame.cpp:149
static void splitAround(Instruction *I, const Twine &Name)
Definition CoroFrame.cpp:1560
static void eliminateSwiftErrorAlloca(Function &F, AllocaInst *Alloca, coro::Shape &Shape)
Eliminate a formerly-swifterror alloca by inserting the get/set intrinsics and attempting to MemToReg...
Definition CoroFrame.cpp:1706
static void movePHIValuesToInsertedBlock(BasicBlock *SuccBB, BasicBlock *InsertedBB, BasicBlock *PredBB, PHINode *UntilPHI=nullptr)
Definition CoroFrame.cpp:1347
static void dumpSpills(StringRef Title, const coro::SpillInfo &Spills)
Definition CoroFrame.cpp:139
static DIType * solveDIType(DIBuilder &Builder, Type *Ty, const DataLayout &Layout, DIScope *Scope, unsigned LineNum, DenseMap< Type *, DIType * > &DITypeCache)
Definition CoroFrame.cpp:601
static bool willLeaveFunctionImmediatelyAfter(BasicBlock *BB, unsigned depth=3)
After we split the coroutine, will the given basic block be along an obvious exit path for the resump...
Definition CoroFrame.cpp:1567
static StructType * buildFrameType(Function &F, coro::Shape &Shape, FrameDataInfo &FrameData, bool OptimizeFrame)
Definition CoroFrame.cpp:860
static void eliminateSwiftErrorArgument(Function &F, Argument &Arg, coro::Shape &Shape, SmallVectorImpl< AllocaInst * > &AllocasToPromote)
"Eliminate" a swifterror argument by reducing it to the alloca case and then loading and storing in t...
Definition CoroFrame.cpp:1732
static void buildFrameDebugInfo(Function &F, coro::Shape &Shape, FrameDataInfo &FrameData)
Build artificial debug info for C++ coroutine frames to allow users to inspect the contents of the fr...
Definition CoroFrame.cpp:688
static BasicBlock * splitBlockIfNotFirst(Instruction *I, const Twine &Name)
Definition CoroFrame.cpp:1547
static void rewritePHIsForCleanupPad(BasicBlock *CleanupPadBB, CleanupPadInst *CleanupPad)
Definition CoroFrame.cpp:1365
static void sinkLifetimeStartMarkers(Function &F, coro::Shape &Shape, SuspendCrossingInfo &Checker, const DominatorTree &DT)
For each local variable that all of its user are only used inside one of suspended region,...
Definition CoroFrame.cpp:1805
static StringRef solveTypeName(Type *Ty)
Create name for Type.
Definition CoroFrame.cpp:563
static Value * emitGetSwiftErrorValue(IRBuilder<> &Builder, Type *ValueTy, coro::Shape &Shape)
Get the current swifterror value.
Definition CoroFrame.cpp:1643
static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape)
Definition CoroFrame.cpp:1003
static bool isLifetimeStart(const Instruction *Inst)
Module.h This file contains the declarations for the Module class.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
uint64_t IntrinsicInst * II
OptimizedStructLayoutField Field
This file provides an interface for laying out a sequence of fields as a struct in a way that attempt...
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallString class.
static const unsigned FramePtr
an instruction to allocate memory on the stack
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
void setSwiftError(bool V)
Specify whether this alloca is used to represent a swifterror.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
PointerType * getType() const
Overload to return most specific pointer type.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
LLVM_ABI bool isArrayAllocation() const
Return true if there is an allocation size parameter to the allocation instruction that is not 1.
void setAlignment(Align Align)
const Value * getArraySize() const
Get the number of elements allocated.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
LLVM_ABI BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
const Instruction & front() const
LLVM_ABI const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
InstListType::iterator iterator
Instruction iterators...
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
This represents the llvm.coro.alloca.alloc instruction.
LLVM_ABI DIDerivedType * createMemberType(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, Metadata *SizeInBits, uint32_t AlignInBits, Metadata *OffsetInBits, DINode::DIFlags Flags, DIType *Ty, DINodeArray Annotations=nullptr)
Create debugging information entry for a member.
LLVM_ABI DIDerivedType * createPointerType(DIType *PointeeTy, uint64_t SizeInBits, uint32_t AlignInBits=0, std::optional< unsigned > DWARFAddressSpace=std::nullopt, StringRef Name="", DINodeArray Annotations=nullptr)
Create debugging information entry for a pointer.
LLVM_ABI DIBasicType * createBasicType(StringRef Name, uint64_t SizeInBits, unsigned Encoding, DINode::DIFlags Flags=DINode::FlagZero, uint32_t NumExtraInhabitants=0, uint32_t DataSizeInBits=0)
Create debugging information entry for a basic type.
LLVM_ABI DICompositeType * createStructType(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber, Metadata *SizeInBits, uint32_t AlignInBits, DINode::DIFlags Flags, DIType *DerivedFrom, DINodeArray Elements, unsigned RunTimeLang=0, DIType *VTableHolder=nullptr, StringRef UniqueIdentifier="", DIType *Specification=nullptr, uint32_t NumExtraInhabitants=0)
Create debugging information entry for a struct.
LLVM_ABI DINodeArray getOrCreateArray(ArrayRef< Metadata * > Elements)
Get a DINodeArray, create one if required.
LLVM_ABI DIExpression * createExpression(ArrayRef< uint64_t > Addr={})
Create a new descriptor for the specified variable which has a complex address expression for its add...
LLVM_ABI DILocalVariable * createAutoVariable(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, DIType *Ty, bool AlwaysPreserve=false, DINode::DIFlags Flags=DINode::FlagZero, uint32_t AlignInBits=0)
Create a new descriptor for an auto variable.
LLVM_ABI void replaceArrays(DICompositeType *&T, DINodeArray Elements, DINodeArray TParams=DINodeArray())
Replace arrays on a composite type.
LLVM_ABI bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static LLVM_ABI DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
LLVM_ABI DIExpression * foldConstantMath()
Try to shorten an expression with constant math operations that can be evaluated at compile time.
LLVM_ABI uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static LLVM_ABI DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
LLVM_ABI bool isSingleLocationExpression() const
Return whether the evaluated expression makes use of a single location at the start of the expression...
Base class for scope-like contexts.
Subprogram description. Uses SubclassData1.
StringRef getName() const
uint64_t getSizeInBits() const
LLVM_ABI uint32_t getAlignInBits() const
A parsed version of the target data layout string in and methods for querying it.
LLVM_ABI const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
LLVM_ABI IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
LLVM_ABI Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
LLVM_ABI void removeFromParent()
DebugLoc getDebugLoc() const
void setDebugLoc(DebugLoc Loc)
LLVM_ABI Function * getFunction()
Record of a variable value-assignment, aka a non instruction representation of the dbg....
LocationType getType() const
LocationType Type
Classification of the debug-info record that this DbgVariableRecord represents.
LLVM_ABI void replaceVariableLocationOp(Value *OldValue, Value *NewValue, bool AllowEmpty=false)
LLVM_ABI Value * getVariableLocationOp(unsigned OpIdx) const
void setExpression(DIExpression *NewExpr)
DIExpression * getExpression() const
bool isDbgDeclareValue() const
Metadata * getRawLocation() const
Returns the metadata operand for the first location description.
bool isDbgDeclare() const
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void insertBefore(InstListType::iterator InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified position.
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
LLVM_ABI void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
This is an important class for using LLVM in a threaded context.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
LLVM_ABI void replaceOperandWith(unsigned I, Metadata *New)
Replace a specific operand.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
LLVMContext & getContext() const
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::string str() const
str - Get the contents as an std::string.
TypeSize getElementOffsetInBits(unsigned Idx) const
Class to represent struct types.
static LLVM_ABI StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
bool isDefinitionAcrossSuspend(BasicBlock *DefBB, User *U) const
void setDefaultDest(BasicBlock *DefaultCase)
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
TinyPtrVector - This class is specialized for cases where there are normally 0 or 1 element in a vect...
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
LLVM_ABI std::string str() const
Return the twine contents as a std::string.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
A Use represents the edge between a Value definition and its users.
LLVM_ABI void set(Value *Val)
User * getUser() const
Returns the User that contains this Use.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
iterator_range< use_iterator > uses()
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
std::function< bool(Instruction &I)> IsMaterializable
virtual void buildCoroutineFrame(bool OptimizeFrame)
Definition CoroFrame.cpp:2071
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
self_iterator getIterator()
A raw_ostream that writes to an SmallVector or SmallString.
StringRef str() const
Return a StringRef for the vector contents.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
@ C
The default llvm calling convention, compatible with C.
@ BasicBlock
Various leaf nodes.
SmallMapVector< Value *, SmallVector< Instruction *, 2 >, 8 > SpillInfo
@ Async
The "async continuation" lowering, where each suspend point creates a single continuation function.
@ RetconOnce
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
@ Retcon
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
BasicBlock::iterator getSpillInsertionPt(const coro::Shape &, Value *Def, const DominatorTree &DT)
bool isSuspendBlock(BasicBlock *BB)
void normalizeCoroutine(Function &F, coro::Shape &Shape, TargetTransformInfo &TTI)
Definition CoroFrame.cpp:2021
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, TargetTransformInfo &TTI, ArrayRef< Value * > Arguments, IRBuilder<> &)
void sinkSpillUsesAfterCoroBegin(const DominatorTree &DT, CoroBeginInst *CoroBegin, coro::SpillInfo &Spills, SmallVectorImpl< coro::AllocaInfo > &Allocas)
Async and Retcon{Once} conventions assume that all spill uses can be sunk after the coro....
LLVM_ABI void doRematerializations(Function &F, SuspendCrossingInfo &Checker, std::function< bool(Instruction &)> IsMaterializable)
void collectSpillsFromArgs(SpillInfo &Spills, Function &F, const SuspendCrossingInfo &Checker)
void collectSpillsFromDbgInfo(SpillInfo &Spills, Function &F, const SuspendCrossingInfo &Checker)
void salvageDebugInfo(SmallDenseMap< Argument *, AllocaInst *, 4 > &ArgToAllocaMap, DbgVariableRecord &DVR, bool UseEntryValue)
Attempts to rewrite the location operand of debug records in terms of the coroutine frame pointer,...
Definition CoroFrame.cpp:1963
void collectSpillsAndAllocasFromInsts(SpillInfo &Spills, SmallVector< AllocaInfo, 8 > &Allocas, SmallVector< Instruction *, 4 > &DeadInstructions, SmallVector< CoroAllocaAllocInst *, 4 > &LocalAllocas, Function &F, const SuspendCrossingInfo &Checker, const DominatorTree &DT, const coro::Shape &Shape)
bool isCPlusPlus(SourceLanguage S)
This is an optimization pass for GlobalISel generic memory operations.
FunctionAddr VTableAddr Value
UnaryFunction for_each(R &&Range, UnaryFunction F)
Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.
MaybeAlign getAlign(const CallInst &I, unsigned Index)
LLVM_ABI void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
decltype(auto) dyn_cast(const From &Val)
dyn_cast - Return the argument parameter cast to the specified type.
unsigned Log2_64_Ceil(uint64_t Value)
Return the ceil log base 2 of the specified value, 64 if the value is zero.
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
auto successors(const MachineBasicBlock *BB)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
auto pred_size(const MachineBasicBlock *BB)
LLVM_ABI bool isAllocaPromotable(const AllocaInst *AI)
Return true if this alloca is legal for promotion.
auto dyn_cast_or_null(const Y &Val)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
LLVM_ABI BasicBlock * ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ, LandingPadInst *OriginalPad=nullptr, PHINode *LandingPadReplacement=nullptr, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
Split the edge connect the specficed blocks in the case that Succ is an Exception Handling Block.
FunctionAddr VTableAddr Count
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
uint64_t offsetToAlignment(uint64_t Value, Align Alignment)
Returns the offset to the next integer (mod 2**64) that is greater than or equal to Value and is a mu...
LLVM_ABI std::pair< uint64_t, Align > performOptimizedStructLayout(MutableArrayRef< OptimizedStructLayoutField > Fields)
Compute a layout for a struct containing the given fields, making a best-effort attempt to minimize t...
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
DWARFExpression::Operation Op
LLVM_ABI TinyPtrVector< DbgVariableRecord * > findDVRDeclareValues(Value *V)
As above, for DVRDeclareValues.
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned BitWidth
LLVM_ABI void updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred, BasicBlock *NewPred, PHINode *Until=nullptr)
Replaces all uses of OldPred with the NewPred block in all PHINodes in a block.
decltype(auto) cast(const From &Val)
cast - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI TinyPtrVector< DbgVariableRecord * > findDVRDeclares(Value *V)
Finds dbg.declare records declaring local variables as living in the memory that 'V' points to.
auto predecessors(const MachineBasicBlock *BB)
LLVM_ABI void setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ)
Sets the unwind edge of an instruction to a particular successor.
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
LLVM_ABI void findDbgUsers(Value *V, SmallVectorImpl< DbgVariableRecord * > &DbgVariableRecords)
Finds the debug info records describing a value.
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align Alignment
The required alignment of this field.
uint64_t Offset
The offset of this field in the final layout.
uint64_t Size
The required size of this field in bytes.
static constexpr uint64_t FlexibleOffset
A special value for Offset indicating that the field can be moved anywhere.
Align getContextAlignment() const
uint64_t ContextHeaderSize
bool IsFrameInlineInStorage
AllocaInst * PromiseAlloca
AsyncLoweringStorage AsyncLowering
AnyCoroIdRetconInst * getRetconCoroId() const
CoroIdInst * getSwitchCoroId() const
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
AllocaInst * getPromiseAlloca() const
SwitchLoweringStorage SwitchLowering
CoroBeginInst * CoroBegin
BasicBlock::iterator getInsertPtAfterFramePtr() const
RetconLoweringStorage RetconLowering
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
SmallVector< CallInst *, 2 > SwiftErrorOps
BasicBlock * AllocaSpillBlock