clang: lib/CodeGen/CGStmtOpenMP.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
30#include "llvm/ADT/SmallSet.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/Frontend/OpenMP/OMPConstants.h"
33#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DebugInfoMetadata.h"
36#include "llvm/IR/Instructions.h"
37#include "llvm/IR/IntrinsicInst.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/Support/AtomicOrdering.h"
40#include "llvm/Support/Debug.h"
41#include
42using namespace clang;
44using namespace llvm::omp;
45
46#define TTL_CODEGEN_TYPE "target-teams-loop-codegen"
47
51
52namespace {
53
54
57 for (const auto *C : S.clauses()) {
59 if (const auto *PreInit =
60 cast_or_null(CPI->getPreInitStmt())) {
61 for (const auto *I : PreInit->decls()) {
62 if (!I->hasAttr()) {
64 } else {
65 CodeGenFunction::AutoVarEmission Emission =
68 }
69 }
70 }
71 }
72 }
73 }
74 CodeGenFunction::OMPPrivateScope InlinedShareds;
75
76 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
79 (isa_and_nonnull(CGF.CurCodeDecl) &&
81 }
82
83public:
84 OMPLexicalScope(
85 CodeGenFunction &CGF, const OMPExecutableDirective &S,
86 const std::optional CapturedRegion = std::nullopt,
87 const bool EmitPreInitStmt = true)
88 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
89 InlinedShareds(CGF) {
90 if (EmitPreInitStmt)
91 emitPreInitStmt(CGF, S);
92 if (!CapturedRegion)
93 return;
94 assert(S.hasAssociatedStmt() &&
95 "Expected associated statement for inlined directive.");
96 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
97 for (const auto &C : CS->captures()) {
98 if (C.capturesVariable() || C.capturesVariableByCopy()) {
99 auto *VD = C.getCapturedVar();
101 "Canonical decl must be captured.");
102 DeclRefExpr DRE(
103 CGF.getContext(), const_cast<VarDecl *>(VD),
105 InlinedShareds.isGlobalVarCaptured(VD)),
108 }
109 }
110 (void)InlinedShareds.Privatize();
111 }
112};
113
114
115
116class OMPParallelScope final : public OMPLexicalScope {
117 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
122 }
123
124public:
125 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
126 : OMPLexicalScope(CGF, S, std::nullopt,
127 EmitPreInitStmt(S)) {}
128};
129
130
131
132class OMPTeamsScope final : public OMPLexicalScope {
133 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
137 }
138
139public:
140 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
141 : OMPLexicalScope(CGF, S, std::nullopt,
142 EmitPreInitStmt(S)) {}
143};
144
145
146
148 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
149 const Stmt *PreInits;
150 CodeGenFunction::OMPMapVars PreCondVars;
151 if (auto *LD = dyn_cast(&S)) {
152 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
153 for (const auto *E : LD->counters()) {
158 }
159
160 for (const auto *C : LD->getClausesOfKind()) {
161 for (const Expr *IRef : C->varlist()) {
162 const auto *OrigVD =
164 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
165 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
167 CGF, OrigVD,
172 }
173 }
174 }
175 (void)PreCondVars.apply(CGF);
176
177 (void)OMPLoopBasedDirective::doForAllLoops(
178 LD->getInnermostCapturedStmt()->getCapturedStmt(),
179 true, LD->getLoopsNumber(),
180 [&CGF](unsigned Cnt, const Stmt *CurStmt) {
181 if (const auto *CXXFor = dyn_cast(CurStmt)) {
182 if (const Stmt *Init = CXXFor->getInit())
183 CGF.EmitStmt(Init);
184 CGF.EmitStmt(CXXFor->getRangeStmt());
185 CGF.EmitStmt(CXXFor->getEndStmt());
186 }
187 return false;
188 });
189 PreInits = LD->getPreInits();
190 } else if (const auto *Tile = dyn_cast(&S)) {
191 PreInits = Tile->getPreInits();
192 } else if (const auto *Stripe = dyn_cast(&S)) {
193 PreInits = Stripe->getPreInits();
194 } else if (const auto *Unroll = dyn_cast(&S)) {
195 PreInits = Unroll->getPreInits();
196 } else if (const auto *Reverse = dyn_cast(&S)) {
197 PreInits = Reverse->getPreInits();
198 } else if (const auto *Interchange =
199 dyn_cast(&S)) {
200 PreInits = Interchange->getPreInits();
201 } else {
202 llvm_unreachable("Unknown loop-based directive kind.");
203 }
204 doEmitPreinits(PreInits);
205 PreCondVars.restore(CGF);
206 }
207
208 void
211 const Stmt *PreInits;
212 if (const auto *Fuse = dyn_cast(&S)) {
213 PreInits = Fuse->getPreInits();
214 } else {
215 llvm_unreachable(
216 "Unknown canonical loop sequence transform directive kind.");
217 }
218 doEmitPreinits(PreInits);
219 }
220
221 void doEmitPreinits(const Stmt *PreInits) {
222 if (PreInits) {
223
224
225
227 if (auto *PreInitCompound = dyn_cast(PreInits))
228 llvm::append_range(PreInitStmts, PreInitCompound->body());
229 else
230 PreInitStmts.push_back(PreInits);
231
232 for (const Stmt *S : PreInitStmts) {
233
234
235 if (auto *PreInitDecl = dyn_cast(S)) {
236 for (Decl *I : PreInitDecl->decls())
238 continue;
239 }
241 }
242 }
243 }
244
245public:
246 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
248 emitPreInitStmt(CGF, S);
249 }
253 emitPreInitStmt(CGF, S);
254 }
255};
256
258 CodeGenFunction::OMPPrivateScope InlinedShareds;
259
260 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
263 (isa_and_nonnull(CGF.CurCodeDecl) &&
265 }
266
267public:
268 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
269 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
270 InlinedShareds(CGF) {
271 for (const auto *C : S.clauses()) {
272 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
273 if (const auto *PreInit =
274 cast_or_null(CPI->getPreInitStmt())) {
275 for (const auto *I : PreInit->decls()) {
276 if (!I->hasAttr()) {
277 CGF.EmitVarDecl(cast(*I));
278 } else {
279 CodeGenFunction::AutoVarEmission Emission =
280 CGF.EmitAutoVarAlloca(cast(*I));
281 CGF.EmitAutoVarCleanups(Emission);
282 }
283 }
284 }
285 } else if (const auto *UDP = dyn_cast(C)) {
286 for (const Expr *E : UDP->varlist()) {
288 if (const auto *OED = dyn_cast(D))
290 }
291 } else if (const auto *UDP = dyn_cast(C)) {
292 for (const Expr *E : UDP->varlist()) {
294 if (const auto *OED = dyn_cast(D))
296 }
297 }
298 }
301 if (const auto *TG = dyn_cast(&S)) {
302 if (const Expr *E = TG->getReductionRef())
304 }
305
306
307 llvm::DenseSet<CanonicalDeclPtr> CopyArrayTemps;
308 for (const auto *C : S.getClausesOfKind()) {
309 if (C->getModifier() != OMPC_REDUCTION_inscan)
310 continue;
311 for (const Expr *E : C->copy_array_temps())
312 CopyArrayTemps.insert(cast(E)->getDecl());
313 }
314 const auto *CS = cast_or_null(S.getAssociatedStmt());
315 while (CS) {
317 if (C.capturesVariable() || C.capturesVariableByCopy()) {
318 auto *VD = C.getCapturedVar();
319 if (CopyArrayTemps.contains(VD))
320 continue;
322 "Canonical decl must be captured.");
323 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
324 isCapturedVar(CGF, VD) ||
326 InlinedShareds.isGlobalVarCaptured(VD)),
328 C.getLocation());
330 }
331 }
333 }
334 (void)InlinedShareds.Privatize();
335 }
336};
337
338}
339
340
341
345 if (Kind != OMPD_loop)
346 return Kind;
347
349 if (const auto *C = S.getSingleClause<OMPBindClause>())
350 BindKind = C->getBindKind();
351
352 switch (BindKind) {
353 case OMPC_BIND_parallel:
354 return OMPD_for;
355 case OMPC_BIND_teams:
356 return OMPD_distribute;
357 case OMPC_BIND_thread:
358 return OMPD_simd;
359 default:
360 return OMPD_loop;
361 }
362}
363
367
369 if (const auto *OrigDRE = dyn_cast(E)) {
370 if (const auto *OrigVD = dyn_cast(OrigDRE->getDecl())) {
371 OrigVD = OrigVD->getCanonicalDecl();
372 bool IsCaptured =
375 (isa_and_nonnull(CurCodeDecl));
377 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
379 }
380 }
382}
383
386 llvm::Value *Size = nullptr;
387 auto SizeInChars = C.getTypeSizeInChars(Ty);
388 if (SizeInChars.isZero()) {
389
390 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
392 Ty = VlaSize.Type;
393 Size =
395 }
396 SizeInChars = C.getTypeSizeInChars(Ty);
397 if (SizeInChars.isZero())
398 return llvm::ConstantInt::get(SizeTy, 0);
399 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
400 }
401 return CGM.getSize(SizeInChars);
402}
403
408 auto CurCap = S.captures().begin();
411 I != E; ++I, ++CurField, ++CurCap) {
412 if (CurField->hasCapturedVLAType()) {
414 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
415 CapturedVars.push_back(Val);
416 } else if (CurCap->capturesThis()) {
417 CapturedVars.push_back(CXXThisValue);
418 } else if (CurCap->capturesVariableByCopy()) {
420
421
422
423 if (!CurField->getType()->isAnyPointerType()) {
427 Twine(CurCap->getCapturedVar()->getName(), ".casted"));
429
433 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
436
437
439
440
442 }
443 CapturedVars.push_back(CV);
444 } else {
445 assert(CurCap->capturesVariable() && "Expected capture by reference.");
447 }
448 }
449}
450
452 QualType DstType, StringRef Name,
453 LValue AddrLV) {
455
457 AddrLV.getAddress().emitRawPointer(CGF), Ctx.getUIntPtrType(),
459
462 return TmpAddr;
463}
464
466 if (T->isLValueReferenceType())
467 return C.getLValueReferenceType(
469 false);
470 if (T->isPointerType())
472 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
473 if (const auto *VLA = dyn_cast(A))
475 if (!A->isVariablyModifiedType())
476 return C.getCanonicalType(T);
477 }
478 return C.getCanonicalParamType(T);
479}
480
481namespace {
482
483struct FunctionOptions {
484
485 const CapturedStmt *S = nullptr;
486
487
488 const bool UIntPtrCastRequired = true;
489
490
491 const bool RegisterCastedArgsOnly = false;
492
493 const StringRef FunctionName;
494
495 SourceLocation Loc;
496 const bool IsDeviceKernel = false;
497 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
498 bool RegisterCastedArgsOnly, StringRef FunctionName,
499 SourceLocation Loc, bool IsDeviceKernel)
500 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
501 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
502 FunctionName(FunctionName), Loc(Loc), IsDeviceKernel(IsDeviceKernel) {}
503};
504}
505
508 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
509 &LocalAddrs,
510 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
511 &VLASizes,
512 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
515 assert(CD->hasBody() && "missing CapturedDecl body");
516
517 CXXThisValue = nullptr;
518
524 TargetArgs.append(
527 auto I = FO.S->captures().begin();
529 if (!FO.UIntPtrCastRequired) {
536 false, false,
537 false);
538 }
540 QualType ArgType = FD->getType();
542 VarDecl *CapVar = nullptr;
543
544
545
546
547
548
549 if (FO.UIntPtrCastRequired &&
550 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
551 I->capturesVariableArrayType()))
553
554 if (I->capturesVariable() || I->capturesVariableByCopy()) {
555 CapVar = I->getCapturedVar();
557 } else if (I->capturesThis()) {
559 } else {
560 assert(I->capturesVariableArrayType());
562 }
563 if (ArgType->isVariablyModifiedType())
568 II, ArgType,
570 } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
572 Ctx, DebugFunctionDecl,
573 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
574 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
575 nullptr, SC_None, nullptr);
576 } else {
579 }
580 Args.emplace_back(Arg);
581
582 TargetArgs.emplace_back(
583 FO.UIntPtrCastRequired
584 ? Arg
586 ++I;
587 }
590 TargetArgs.append(
593
594
596 FO.IsDeviceKernel
598 TargetArgs)
600 TargetArgs);
602
603 auto *F =
604 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
605 FO.FunctionName, &CGM.getModule());
608 F->setDoesNotThrow();
609 F->setDoesNotRecurse();
610
611
613 F->removeFnAttr(llvm::Attribute::NoInline);
614 F->addFnAttr(llvm::Attribute::AlwaysInline);
615 }
616
617
619 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
620 FO.UIntPtrCastRequired ? FO.Loc
623 I = FO.S->captures().begin();
625
627 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
629 TargetArgs[Cnt]);
630 } else {
632 }
633
634
635 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
636 const VarDecl *CurVD = I->getCapturedVar();
637 if (!FO.RegisterCastedArgsOnly)
638 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
639 ++Cnt;
640 ++I;
641 continue;
642 }
643
646 if (FD->hasCapturedVLAType()) {
647 if (FO.UIntPtrCastRequired) {
650 Args[Cnt]->getName(), ArgLVal),
652 }
653 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
655 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
656 } else if (I->capturesVariable()) {
657 const VarDecl *Var = I->getCapturedVar();
659 Address ArgAddr = ArgLVal.getAddress();
660 if (ArgLVal.getType()->isLValueReferenceType()) {
663 assert(ArgLVal.getType()->isPointerType());
665 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
666 }
667 if (!FO.RegisterCastedArgsOnly) {
668 LocalAddrs.insert(
670 }
671 } else if (I->capturesVariableByCopy()) {
672 assert(!FD->getType()->isAnyPointerType() &&
673 "Not expecting a captured pointer.");
674 const VarDecl *Var = I->getCapturedVar();
675 LocalAddrs.insert({Args[Cnt],
676 {Var, FO.UIntPtrCastRequired
678 CGF, I->getLocation(), FD->getType(),
679 Args[Cnt]->getName(), ArgLVal)
680 : ArgLVal.getAddress()}});
681 } else {
682
683 assert(I->capturesThis());
684 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
685 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
686 }
687 ++Cnt;
688 ++I;
689 }
690
691 return F;
692}
693
697 assert(
699 "CapturedStmtInfo should be set when generating the captured function");
701
702 bool NeedWrapperFunction =
703 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
705 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs,
706 WrapperLocalAddrs;
707 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes,
708 WrapperVLASizes;
710 llvm::raw_svector_ostream Out(Buffer);
713 bool IsDeviceKernel = CGM.getOpenMPRuntime().isGPU() &&
715 D.getCapturedStmt(OMPD_target) == &S;
716 CodeGenFunction WrapperCGF(CGM, true);
717 llvm::Function *WrapperF = nullptr;
718 if (NeedWrapperFunction) {
719
720
721 FunctionOptions WrapperFO(&S, true,
722 true,
724 IsDeviceKernel);
726 WrapperF =
728 WrapperCGF.CXXThisValue, WrapperFO);
729 Out << "_debug__";
730 }
731 FunctionOptions FO(&S, !NeedWrapperFunction, false,
732 Out.str(), Loc, !NeedWrapperFunction && IsDeviceKernel);
734 *this, WrapperArgs, WrapperLocalAddrs, WrapperVLASizes, CXXThisValue, FO);
736 for (const auto &LocalAddrPair : WrapperLocalAddrs) {
737 if (LocalAddrPair.second.first) {
738 LocalScope.addPrivate(LocalAddrPair.second.first,
739 LocalAddrPair.second.second);
740 }
741 }
742 (void)LocalScope.Privatize();
743 for (const auto &VLASizePair : WrapperVLASizes)
744 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
745 PGO->assignRegionCounters(GlobalDecl(CD), F);
747 LocalScope.ForceCleanup();
749 if (!NeedWrapperFunction)
750 return F;
751
752
753 WrapperF->removeFromParent();
754 F->getParent()->getFunctionList().insertAfter(F->getIterator(), WrapperF);
755
757 auto *PI = F->arg_begin();
758 for (const auto *Arg : Args) {
760 auto I = LocalAddrs.find(Arg);
761 if (I != LocalAddrs.end()) {
763 I->second.second,
764 I->second.first ? I->second.first->getType() : Arg->getType(),
769 } else {
770 auto EI = VLASizes.find(Arg);
771 if (EI != VLASizes.end()) {
772 CallArg = EI->second.second;
773 } else {
778 }
779 }
781 ++PI;
782 }
783 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
785 return WrapperF;
786}
787
788
789
790
793 const llvm::function_ref<void(Address, Address)> CopyGen) {
794
796
797
799 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
801
802 llvm::Value *SrcBegin = SrcAddr.emitRawPointer(*this);
803 llvm::Value *DestBegin = DestAddr.emitRawPointer(*this);
804
806 DestBegin, NumElements);
807
808
809 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
810 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
811 llvm::Value *IsEmpty =
812 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
813 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
814
815
816 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
818
820
821 llvm::PHINode *SrcElementPHI =
822 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
823 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
824 Address SrcElementCurrent =
827
828 llvm::PHINode *DestElementPHI = Builder.CreatePHI(
829 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
830 DestElementPHI->addIncoming(DestBegin, EntryBB);
831 Address DestElementCurrent =
834
835
836 CopyGen(DestElementCurrent, SrcElementCurrent);
837
838
839 llvm::Value *DestElementNext =
841 1, "omp.arraycpy.dest.element");
842 llvm::Value *SrcElementNext =
844 1, "omp.arraycpy.src.element");
845
846 llvm::Value *Done =
847 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
848 Builder.CreateCondBr(Done, DoneBB, BodyBB);
849 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
850 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
851
852
853 EmitBlock(DoneBB, true);
854}
855
860 const auto *BO = dyn_cast(Copy);
861 if (BO && BO->getOpcode() == BO_Assign) {
862
866 } else {
867
868
870 DestAddr, SrcAddr, OriginalType,
871 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
872
873
874
876 Remap.addPrivate(DestVD, DestElement);
880 });
881 }
882 } else {
883
888
890 }
891}
892
896 return false;
898 bool DeviceConstTarget = getLangOpts().OpenMPIsTargetDevice &&
900 bool FirstprivateIsLastprivate = false;
901 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
903 for (const auto *D : C->varlist())
904 Lastprivates.try_emplace(
906 C->getKind());
907 }
908 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
911
912
913 bool MustEmitFirstprivateCopy =
914 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
916 const auto *IRef = C->varlist_begin();
917 const auto *InitsRef = C->inits().begin();
918 for (const Expr *IInit : C->private_copies()) {
920 bool ThisFirstprivateIsLastprivate =
921 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
924 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
926 (!VD || !VD->hasAttr())) {
927 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
928 ++IRef;
929 ++InitsRef;
930 continue;
931 }
932
933
934 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
936 (!VD || !VD->hasAttr())) {
937 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
938 ++IRef;
939 ++InitsRef;
940 continue;
941 }
942 FirstprivateIsLastprivate =
943 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
944 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
945 const auto *VDInit =
947 bool IsRegistered;
949 FD != nullptr,
952 if (!FD) {
953
956
957 ++IRef;
958 ++InitsRef;
959 continue;
960 }
963 } else {
964 assert(!CE && "Expected non-constant firstprivate.");
966 }
967 } else {
969 }
971 if (Type->isArrayType()) {
972
973
974
978
981 } else {
985
986
987 RunCleanupsScope InitScope(*this);
988
989 setAddrOfLocalVar(VDInit, SrcElement);
990 EmitAnyExprToMem(Init, DestElement,
991 Init->getType().getQualifiers(),
992 false);
993 LocalDeclMap.erase(VDInit);
994 });
995 }
997 IsRegistered =
999 } else {
1001
1002
1003
1004 setAddrOfLocalVar(VDInit, OriginalAddr);
1006 LocalDeclMap.erase(VDInit);
1008 if (ThisFirstprivateIsLastprivate &&
1009 Lastprivates[OrigVD->getCanonicalDecl()] ==
1010 OMPC_LASTPRIVATE_conditional) {
1011
1012 llvm::Value *V =
1015 (*IRef)->getExprLoc());
1016 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1017 *this, OrigVD);
1020 LocalDeclMap.erase(VD);
1021 setAddrOfLocalVar(VD, VDAddr);
1022 }
1023 IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1024 }
1025 assert(IsRegistered &&
1026 "firstprivate var already registered as private");
1027
1028 (void)IsRegistered;
1029 }
1030 ++IRef;
1031 ++InitsRef;
1032 }
1033 }
1034 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
1035}
1036
1041 return;
1042 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
1044 auto IRef = C->varlist_begin();
1045 for (const Expr *IInit : C->private_copies()) {
1047 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
1050
1051 bool IsRegistered =
1053 assert(IsRegistered && "private var already registered as private");
1054
1055 (void)IsRegistered;
1056 }
1057 ++IRef;
1058 }
1059 }
1060}
1061
1064 return false;
1065
1066
1067
1068
1069 llvm::DenseSet<const VarDecl *> CopiedVars;
1070 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
1071 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
1072 auto IRef = C->varlist_begin();
1073 auto ISrcRef = C->source_exprs().begin();
1074 auto IDestRef = C->destination_exprs().begin();
1075 for (const Expr *AssignOp : C->assignment_ops()) {
1079
1080
1081
1084 getContext().getTargetInfo().isTLSSupported()) {
1086 "Copyin threadprivates should have been captured!");
1090 LocalDeclMap.erase(VD);
1091 } else {
1092 MasterAddr =
1094 : CGM.GetAddrOfGlobal(VD),
1095 CGM.getTypes().ConvertTypeForMem(VD->getType()),
1097 }
1098
1100 if (CopiedVars.size() == 1) {
1101
1102
1105
1106 auto *MasterAddrInt = Builder.CreatePtrToInt(
1108 auto *PrivateAddrInt = Builder.CreatePtrToInt(
1111 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1112 CopyEnd);
1114 }
1115 const auto *SrcVD =
1117 const auto *DestVD =
1119 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
1120 }
1121 ++IRef;
1122 ++ISrcRef;
1123 ++IDestRef;
1124 }
1125 }
1126 if (CopyEnd) {
1127
1128 EmitBlock(CopyEnd, true);
1129 return true;
1130 }
1131 return false;
1132}
1133
1137 return false;
1138 bool HasAtLeastOneLastprivate = false;
1140 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1143 for (const Expr *C : LoopDirective->counters()) {
1144 SIMDLCVs.insert(
1146 }
1147 }
1148 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1150 HasAtLeastOneLastprivate = true;
1152 break;
1153 const auto *IRef = C->varlist_begin();
1154 const auto *IDestRef = C->destination_exprs().begin();
1155 for (const Expr *IInit : C->private_copies()) {
1156
1157
1159
1160
1161 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1162 const auto *DestVD =
1165
1167 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1169
1170
1171
1172 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1175 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
1176 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1177 *this, OrigVD);
1178 setAddrOfLocalVar(VD, VDAddr);
1179 } else {
1180
1183 }
1184 bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1185 assert(IsRegistered &&
1186 "lastprivate var already registered as private");
1187 (void)IsRegistered;
1188 }
1189 }
1190 ++IRef;
1191 ++IDestRef;
1192 }
1193 }
1194 return HasAtLeastOneLastprivate;
1195}
1196
1199 llvm::Value *IsLastIterCond) {
1201 return;
1202
1203
1204
1205
1206
1207
1208 llvm::BasicBlock *ThenBB = nullptr;
1209 llvm::BasicBlock *DoneBB = nullptr;
1210 if (IsLastIterCond) {
1211
1212
1216 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1217 })) {
1218 CGM.getOpenMPRuntime().emitBarrierCall(*this, D.getBeginLoc(),
1219 OMPD_unknown,
1220 false,
1221 true);
1222 }
1225 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1227 }
1228 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1229 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1230 if (const auto *LoopDirective = dyn_cast(&D)) {
1231 auto IC = LoopDirective->counters().begin();
1232 for (const Expr *F : LoopDirective->finals()) {
1233 const auto *D =
1235 if (NoFinals)
1236 AlreadyEmittedVars.insert(D);
1237 else
1238 LoopCountersAndUpdates[D] = F;
1239 ++IC;
1240 }
1241 }
1243 auto IRef = C->varlist_begin();
1244 auto ISrcRef = C->source_exprs().begin();
1245 auto IDestRef = C->destination_exprs().begin();
1246 for (const Expr *AssignOp : C->assignment_ops()) {
1247 const auto *PrivateVD =
1250 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1251 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1252
1253
1254
1255 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1257 const auto *SrcVD =
1259 const auto *DestVD =
1261
1263 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1265 Builder.CreateLoad(PrivateAddr),
1266 CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),
1267 CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
1268
1269 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1270 CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
1272 (*IRef)->getExprLoc());
1273
1275 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1276 }
1277 ++IRef;
1278 ++ISrcRef;
1279 ++IDestRef;
1280 }
1281 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1283 }
1284 if (IsLastIterCond)
1285 EmitBlock(DoneBB, true);
1286}
1287
1292 return;
1302 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
1303 continue;
1304 Shareds.append(C->varlist_begin(), C->varlist_end());
1305 Privates.append(C->privates().begin(), C->privates().end());
1306 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1307 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1308 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1309 if (C->getModifier() == OMPC_REDUCTION_task) {
1310 Data.ReductionVars.append(C->privates().begin(), C->privates().end());
1311 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
1312 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
1313 Data.ReductionOps.append(C->reduction_ops().begin(),
1314 C->reduction_ops().end());
1315 TaskLHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1316 TaskRHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1317 }
1318 }
1319 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
1320 unsigned Count = 0;
1321 auto *ILHS = LHSs.begin();
1322 auto *IRHS = RHSs.begin();
1323 auto *IPriv = Privates.begin();
1324 for (const Expr *IRef : Shareds) {
1326
1332 [&Emission](CodeGenFunction &CGF) {
1333 CGF.EmitAutoVarInit(Emission);
1334 return true;
1335 });
1339 bool IsRegistered =
1341 assert(IsRegistered && "private var already registered as private");
1342
1343 (void)IsRegistered;
1344
1349 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1350
1351
1354 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1356
1357
1362 } else {
1366
1367
1368 if (IsArray) {
1369 OriginalAddr =
1371 }
1372 PrivateScope.addPrivate(LHSVD, OriginalAddr);
1377 }
1378 ++ILHS;
1379 ++IRHS;
1380 ++IPriv;
1381 ++Count;
1382 }
1383 if (.ReductionVars.empty()) {
1385 Data.IsReductionWithTaskMod = true;
1387 llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(
1388 *this, D.getBeginLoc(), TaskLHSs, TaskRHSs, Data);
1389 const Expr *TaskRedRef = nullptr;
1390 switch (EKind) {
1391 case OMPD_parallel:
1393 break;
1394 case OMPD_for:
1396 break;
1397 case OMPD_sections:
1399 break;
1400 case OMPD_parallel_for:
1402 break;
1403 case OMPD_parallel_master:
1404 TaskRedRef =
1406 break;
1407 case OMPD_parallel_sections:
1408 TaskRedRef =
1410 break;
1411 case OMPD_target_parallel:
1412 TaskRedRef =
1414 break;
1415 case OMPD_target_parallel_for:
1416 TaskRedRef =
1418 break;
1419 case OMPD_distribute_parallel_for:
1420 TaskRedRef =
1422 break;
1423 case OMPD_teams_distribute_parallel_for:
1425 .getTaskReductionRefExpr();
1426 break;
1427 case OMPD_target_teams_distribute_parallel_for:
1429 .getTaskReductionRefExpr();
1430 break;
1431 case OMPD_simd:
1432 case OMPD_for_simd:
1433 case OMPD_section:
1434 case OMPD_single:
1435 case OMPD_master:
1436 case OMPD_critical:
1437 case OMPD_parallel_for_simd:
1438 case OMPD_task:
1439 case OMPD_taskyield:
1440 case OMPD_error:
1441 case OMPD_barrier:
1442 case OMPD_taskwait:
1443 case OMPD_taskgroup:
1444 case OMPD_flush:
1445 case OMPD_depobj:
1446 case OMPD_scan:
1447 case OMPD_ordered:
1448 case OMPD_atomic:
1449 case OMPD_teams:
1450 case OMPD_target:
1451 case OMPD_cancellation_point:
1452 case OMPD_cancel:
1453 case OMPD_target_data:
1454 case OMPD_target_enter_data:
1455 case OMPD_target_exit_data:
1456 case OMPD_taskloop:
1457 case OMPD_taskloop_simd:
1458 case OMPD_master_taskloop:
1459 case OMPD_master_taskloop_simd:
1460 case OMPD_parallel_master_taskloop:
1461 case OMPD_parallel_master_taskloop_simd:
1462 case OMPD_distribute:
1463 case OMPD_target_update:
1464 case OMPD_distribute_parallel_for_simd:
1465 case OMPD_distribute_simd:
1466 case OMPD_target_parallel_for_simd:
1467 case OMPD_target_simd:
1468 case OMPD_teams_distribute:
1469 case OMPD_teams_distribute_simd:
1470 case OMPD_teams_distribute_parallel_for_simd:
1471 case OMPD_target_teams:
1472 case OMPD_target_teams_distribute:
1473 case OMPD_target_teams_distribute_parallel_for_simd:
1474 case OMPD_target_teams_distribute_simd:
1475 case OMPD_declare_target:
1476 case OMPD_end_declare_target:
1477 case OMPD_threadprivate:
1478 case OMPD_allocate:
1479 case OMPD_declare_reduction:
1480 case OMPD_declare_mapper:
1481 case OMPD_declare_simd:
1482 case OMPD_requires:
1483 case OMPD_declare_variant:
1484 case OMPD_begin_declare_variant:
1485 case OMPD_end_declare_variant:
1486 case OMPD_unknown:
1487 default:
1488 llvm_unreachable("Unexpected directive with task reductions.");
1489 }
1490
1494 false, TaskRedRef->getType());
1495 }
1496}
1497
1501 return;
1507 bool HasAtLeastOneReduction = false;
1508 bool IsReductionWithTaskMod = false;
1510
1511 if (C->getModifier() == OMPC_REDUCTION_inscan)
1512 continue;
1513 HasAtLeastOneReduction = true;
1514 Privates.append(C->privates().begin(), C->privates().end());
1515 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1516 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1517 IsPrivateVarReduction.append(C->private_var_reduction_flags().begin(),
1518 C->private_var_reduction_flags().end());
1519 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1520 IsReductionWithTaskMod =
1521 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
1522 }
1523 if (HasAtLeastOneReduction) {
1525 if (IsReductionWithTaskMod) {
1526 CGM.getOpenMPRuntime().emitTaskReductionFini(
1528 }
1529 bool TeamsLoopCanBeParallel = false;
1530 if (auto *TTLD = dyn_cast(&D))
1531 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();
1532 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1534 TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;
1535 bool SimpleReduction = ReductionKind == OMPD_simd;
1536
1537
1538 CGM.getOpenMPRuntime().emitReduction(
1539 *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1540 {WithNowait, SimpleReduction, IsPrivateVarReduction, ReductionKind});
1541 }
1542}
1543
1546 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1548 return;
1549 llvm::BasicBlock *DoneBB = nullptr;
1551 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1552 if (!DoneBB) {
1553 if (llvm::Value *Cond = CondGen(CGF)) {
1554
1555
1556 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1558 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1560 }
1561 }
1563 }
1564 }
1565 if (DoneBB)
1566 CGF.EmitBlock(DoneBB, true);
1567}
1568
1569namespace {
1570
1571
1572
1574 const OMPExecutableDirective &,
1575 llvm::SmallVectorImpl<llvm::Value *> &)>
1576 CodeGenBoundParametersTy;
1577}
1578
1579static void
1583 return;
1584 llvm::DenseSet<CanonicalDeclPtr> PrivateDecls;
1586 for (const Expr *Ref : C->varlist()) {
1587 if (!Ref->getType()->isScalarType())
1588 continue;
1589 const auto *DRE = dyn_cast(Ref->IgnoreParenImpCasts());
1590 if (!DRE)
1591 continue;
1592 PrivateDecls.insert(cast(DRE->getDecl()));
1594 }
1595 }
1597 for (const Expr *Ref : C->varlist()) {
1598 if (!Ref->getType()->isScalarType())
1599 continue;
1600 const auto *DRE = dyn_cast(Ref->IgnoreParenImpCasts());
1601 if (!DRE)
1602 continue;
1603 PrivateDecls.insert(cast(DRE->getDecl()));
1605 }
1606 }
1607 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
1608 for (const Expr *Ref : C->varlist()) {
1609 if (!Ref->getType()->isScalarType())
1610 continue;
1611 const auto *DRE = dyn_cast(Ref->IgnoreParenImpCasts());
1612 if (!DRE)
1613 continue;
1614 PrivateDecls.insert(cast(DRE->getDecl()));
1616 }
1617 }
1618
1619
1620
1621
1623 for (const Expr *Ref : C->varlist()) {
1624 if (!Ref->getType()->isScalarType())
1625 continue;
1626 const auto *DRE = dyn_cast(Ref->IgnoreParenImpCasts());
1627 if (!DRE)
1628 continue;
1629 PrivateDecls.insert(cast(DRE->getDecl()));
1630 }
1631 }
1633 CGF, S, PrivateDecls);
1634}
1635
1639 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1640 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1641 llvm::Value *NumThreads = nullptr;
1643
1644
1649
1650 llvm::Function *OutlinedFn =
1654
1655 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1657 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1658 true);
1659 Modifier = NumThreadsClause->getModifier();
1660 if (const auto *MessageClause = S.getSingleClause<OMPMessageClause>()) {
1661 Message = MessageClause->getMessageString();
1662 MessageLoc = MessageClause->getBeginLoc();
1663 }
1664 if (const auto *SeverityClause = S.getSingleClause<OMPSeverityClause>()) {
1665 Severity = SeverityClause->getSeverityKind();
1666 SeverityLoc = SeverityClause->getBeginLoc();
1667 }
1669 CGF, NumThreads, NumThreadsClause->getBeginLoc(), Modifier, Severity,
1670 SeverityLoc, Message, MessageLoc);
1671 }
1672 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1675 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1676 }
1677 const Expr *IfCond = nullptr;
1678 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1679 if (C->getNameModifier() == OMPD_unknown ||
1680 C->getNameModifier() == OMPD_parallel) {
1681 IfCond = C->getCondition();
1682 break;
1683 }
1684 }
1685
1686 OMPParallelScope Scope(CGF, S);
1688
1689
1690
1691
1692 CodeGenBoundParameters(CGF, S, CapturedVars);
1695 CapturedVars, IfCond, NumThreads,
1696 Modifier, Severity, Message);
1697}
1698
1701 if (!CVD->hasAttr())
1702 return false;
1703 const auto *AA = CVD->getAttr();
1704
1705 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1706 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1707 !AA->getAllocator());
1708}
1709
1713
1717 if (Copyins) {
1718
1719
1720
1722 CGF, S.getBeginLoc(), OMPD_unknown, false,
1723 true);
1724 }
1725}
1726
1728 CodeGenFunction &CGF, const VarDecl *VD) {
1730 auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1731
1732 if (!VD)
1737 llvm::Value *Size;
1738 CharUnits Align = CGM.getContext().getDeclAlign(CVD);
1741
1742 Size = CGF.Builder.CreateNUWAdd(
1744 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
1745 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
1746 } else {
1748 Size = CGM.getSize(Sz.alignTo(Align));
1749 }
1750
1751 const auto *AA = CVD->getAttr();
1752 assert(AA->getAllocator() &&
1753 "Expected allocator expression for non-default allocator.");
1754 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
1755
1756
1757 if (Allocator->getType()->isIntegerTy())
1758 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
1759 else if (Allocator->getType()->isPointerTy())
1761 CGM.VoidPtrTy);
1762
1763 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
1764 CGF.Builder, Size, Allocator,
1766 llvm::CallInst *FreeCI =
1767 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator);
1768
1775}
1776
1778 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
1781 if (CGM.getLangOpts().OpenMPUseTLS &&
1782 CGM.getContext().getTargetInfo().isTLSSupported())
1783 return VDAddr;
1784
1785 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1786
1788 llvm::Value *Data =
1790 llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy));
1792 llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix);
1793
1794 llvm::CallInst *ThreadPrivateCacheCall =
1795 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);
1796
1798}
1799
1801 ArrayRef Parts, StringRef FirstSeparator, StringRef Separator) {
1803 llvm::raw_svector_ostream OS(Buffer);
1804 StringRef Sep = FirstSeparator;
1805 for (StringRef Part : Parts) {
1806 OS << Sep << Part;
1807 Sep = Separator;
1808 }
1809 return OS.str().str();
1810}
1811
1813 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1816 Builder.restoreIP(CodeGenIP);
1817 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, false,
1818 "." + RegionName + ".after");
1819
1820 {
1822 CGF.EmitStmt(RegionBodyStmt);
1823 }
1824
1825 if (Builder.saveIP().isSet())
1826 Builder.CreateBr(FiniBB);
1827}
1828
1830 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1833 Builder.restoreIP(CodeGenIP);
1834 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, false,
1835 "." + RegionName + ".after");
1836
1837 {
1839 CGF.EmitStmt(RegionBodyStmt);
1840 }
1841
1842 if (Builder.saveIP().isSet())
1843 Builder.CreateBr(FiniBB);
1844}
1845
1847 if (CGM.getLangOpts().OpenMPIRBuilder) {
1848 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1849
1850 llvm::Value *IfCond = nullptr;
1851 if (const auto *C = S.getSingleClause<OMPIfClause>())
1853 true);
1854
1855 llvm::Value *NumThreads = nullptr;
1856 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())
1857 NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(),
1858 true);
1859
1860 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1861 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
1862 ProcBind = ProcBindClause->getProcBindKind();
1863
1864 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1865
1866
1867
1868 auto FiniCB = [this](InsertPointTy IP) {
1870 return llvm::Error::success();
1871 };
1872
1873
1874
1875
1876
1877 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1878 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1879
1880
1881 ReplVal = &Val;
1882
1883 return CodeGenIP;
1884 };
1885
1888
1889 auto BodyGenCB = [&, this](InsertPointTy AllocaIP,
1890 InsertPointTy CodeGenIP) {
1892 *this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP, "parallel");
1893 return llvm::Error::success();
1894 };
1895
1898 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1900 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
1901 OMPBuilder.createParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1902 IfCond, NumThreads, ProcBind, S.hasCancel()));
1903 Builder.restoreIP(AfterIP);
1904 return;
1905 }
1906
1907
1909 Action.Enter(CGF);
1916 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1918 };
1919 {
1920 auto LPCRegion =
1925 [](CodeGenFunction &) { return nullptr; });
1926 }
1927
1929}
1930
1934
1935namespace {
1936
1937class OMPTransformDirectiveScopeRAII {
1938 OMPLoopScope *Scope = nullptr;
1941
1942 OMPTransformDirectiveScopeRAII(const OMPTransformDirectiveScopeRAII &) =
1943 delete;
1944 OMPTransformDirectiveScopeRAII &
1945 operator=(const OMPTransformDirectiveScopeRAII &) = delete;
1946
1947public:
1949 if (const auto *Dir = dyn_cast(S)) {
1950 Scope = new OMPLoopScope(CGF, *Dir);
1953 } else if (const auto *Dir =
1954 dyn_cast(
1955 S)) {
1956
1957
1958
1959 Scope = new OMPLoopScope(CGF, *Dir);
1962 }
1963 }
1964 ~OMPTransformDirectiveScopeRAII() {
1965 if (!Scope)
1966 return;
1967 delete CapInfoRAII;
1968 delete CGSI;
1969 delete Scope;
1970 }
1971};
1972}
1973
1975 int MaxLevel, int Level = 0) {
1976 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
1978 if (const auto *CS = dyn_cast(SimplifiedS)) {
1981 "LLVM IR generation of compound statement ('{}')");
1982
1983
1985 for (const Stmt *CurStmt : CS->body())
1986 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1987 return;
1988 }
1989 if (SimplifiedS == NextLoop) {
1990 if (auto *Dir = dyn_cast(SimplifiedS))
1991 SimplifiedS = Dir->getTransformedStmt();
1992 if (const auto *CanonLoop = dyn_cast(SimplifiedS))
1993 SimplifiedS = CanonLoop->getLoopStmt();
1994 if (const auto *For = dyn_cast(SimplifiedS)) {
1995 S = For->getBody();
1996 } else {
1998 "Expected canonical for loop or range-based for loop.");
2000 CGF.EmitStmt(CXXFor->getLoopVarStmt());
2001 S = CXXFor->getBody();
2002 }
2003 if (Level + 1 < MaxLevel) {
2004 NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(
2005 S, true);
2006 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
2007 return;
2008 }
2009 }
2011}
2012
2016
2017 for (const Expr *UE : D.updates())
2019
2020
2021
2024 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2025 for (const Expr *UE : C->updates())
2027 }
2028 }
2029
2030
2032 BreakContinueStack.push_back(BreakContinue(D, LoopExit, Continue));
2033 for (const Expr *E : D.finals_conditions()) {
2034 if (!E)
2035 continue;
2036
2037
2038 llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");
2042 }
2043
2046 bool IsInscanRegion = InscanScope.Privatize();
2047 if (IsInscanRegion) {
2048
2049
2050
2051
2052
2055
2056
2057 if (EKind != OMPD_simd && ().OpenMPSimd)
2062 }
2063
2064
2065 const Stmt *Body =
2066 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
2067
2069 OMPLoopBasedDirective::tryToFindNextInnerLoop(
2070 Body, true),
2071 D.getLoopsNumber());
2072
2073
2074 if (IsInscanRegion)
2076
2077
2079 BreakContinueStack.pop_back();
2080}
2081
2083
2084
2085
2090 std::unique_ptrCodeGenFunction::CGCapturedStmtInfo CSI =
2091 std::make_uniqueCodeGenFunction::CGCapturedStmtInfo(*S);
2094
2095 return {F, CapStruct.getPointer(ParentCGF)};
2096}
2097
2098
2099static llvm::CallInst *
2102
2104 EffectiveArgs.reserve(Args.size() + 1);
2105 llvm::append_range(EffectiveArgs, Args);
2106 EffectiveArgs.push_back(Cap.second);
2107
2108 return ParentCGF.Builder.CreateCall(Cap.first, EffectiveArgs);
2109}
2110
2111llvm::CanonicalLoopInfo *
2113 assert(Depth == 1 && "Nested loops with OpenMPIRBuilder not yet implemented");
2114
2115
2116
2117
2118
2119
2120
2123
2125 assert(OMPLoopNestStack.size() >= (size_t)Depth && "Found too few loops");
2126
2127
2129
2130
2131
2134
2136}
2137
2139 const Stmt *SyntacticalLoop = S->getLoopStmt();
2141
2143 return;
2144 }
2145
2146 LexicalScope ForScope(*this, S->getSourceRange());
2147
2148
2149
2150 const Stmt *BodyStmt;
2151 if (const auto *For = dyn_cast(SyntacticalLoop)) {
2152 if (const Stmt *InitStmt = For->getInit())
2154 BodyStmt = For->getBody();
2155 } else if (const auto *RangeFor =
2156 dyn_cast(SyntacticalLoop)) {
2157 if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2159 if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2161 if (const DeclStmt *EndStmt = RangeFor->getEndStmt())
2163 if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2165 BodyStmt = RangeFor->getBody();
2166 } else
2167 llvm_unreachable("Expected for-stmt or range-based for-stmt");
2168
2169
2170 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2172 const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();
2174
2175
2176
2183 llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count");
2184
2185
2186 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2187 auto BodyGen = [&, this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2188 llvm::Value *IndVar) {
2189 Builder.restoreIP(CodeGenIP);
2190
2191
2192
2193 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2198
2201 return llvm::Error::success();
2202 };
2203
2204 llvm::CanonicalLoopInfo *CL =
2205 cantFail(OMPBuilder.createCanonicalLoop(Builder, BodyGen, DistVal));
2206
2207
2208 Builder.restoreIP(CL->getAfterIP());
2210
2211
2213}
2214
2217 const Expr *IncExpr,
2218 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
2219 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
2221
2222
2225 const SourceRange R = S.getSourceRange();
2226
2227
2229 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2231 const AttributedStmt *AS = dyn_cast_or_null(SS);
2233 if (AS)
2234 LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(),
2237 else
2240
2241
2242
2243 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2244 if (RequiresCleanup)
2246
2247 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
2248
2249
2251 if (ExitBlock != LoopExit.getBlock()) {
2254 }
2255
2258
2259
2261 BreakContinueStack.push_back(BreakContinue(S, LoopExit, Continue));
2262
2263 BodyGen(*this);
2264
2265
2268 PostIncGen(*this);
2269 BreakContinueStack.pop_back();
2272
2274}
2275
2278 return false;
2279
2280 bool HasLinears = false;
2281 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2282 for (const Expr *Init : C->inits()) {
2283 HasLinears = true;
2285 if (const auto *Ref =
2288 const auto *OrigVD = cast(Ref->getDecl());
2294 &DRE, VD,
2296 false);
2298 } else {
2300 }
2301 }
2302
2303
2304 if (const auto *CS = cast_or_null(C->getCalcStep()))
2307
2309 }
2310 }
2311 return HasLinears;
2312}
2313
2316 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2318 return;
2319 llvm::BasicBlock *DoneBB = nullptr;
2320
2321 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2322 auto IC = C->varlist_begin();
2323 for (const Expr *F : C->finals()) {
2324 if (!DoneBB) {
2325 if (llvm::Value *Cond = CondGen(*this)) {
2326
2327
2328 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
2330 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2332 }
2333 }
2337 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
2340 VarScope.addPrivate(OrigVD, OrigAddr);
2343 ++IC;
2344 }
2345 if (const Expr *PostUpdate = C->getPostUpdateExpr())
2347 }
2348 if (DoneBB)
2349 EmitBlock(DoneBB, true);
2350}
2351
2355 return;
2356 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
2357 llvm::APInt ClauseAlignment(64, 0);
2358 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2359 auto *AlignmentCI =
2361 ClauseAlignment = AlignmentCI->getValue();
2362 }
2363 for (const Expr *E : Clause->varlist()) {
2364 llvm::APInt Alignment(ClauseAlignment);
2365 if (Alignment == 0) {
2366
2367
2368
2369 Alignment =
2372 E->getType()->getPointeeType()))
2374 }
2375 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2376 "alignment is not power of 2");
2377 if (Alignment != 0) {
2380 PtrValue, E, SourceLocation(),
2381 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
2382 }
2383 }
2384 }
2385}
2386
2390 return;
2391 auto I = S.private_counters().begin();
2392 for (const Expr *E : S.counters()) {
2395
2398 LocalDeclMap.erase(PrivateVD);
2400 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
2404 E->getType(), VK_LValue, E->getExprLoc());
2406 } else {
2408 }
2409 ++I;
2410 }
2411
2413 if (->getNumForLoops())
2414 continue;
2415 for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
2416 I < E; ++I) {
2419
2420
2421 if (DRE->refersToEnclosingVariableOrCapture()) {
2424 }
2425 }
2426 }
2427}
2428
2430 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2431 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2433 return;
2434 {
2438
2439 for (const Expr *I : S.inits()) {
2441 }
2442 }
2443
2444
2446 for (const Expr *E : S.dependent_counters()) {
2447 if (!E)
2448 continue;
2449 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2450 "dependent counter must not be an iterator.");
2454 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2455 }
2456 (void)PreCondVars.apply(CGF);
2457 for (const Expr *E : S.dependent_inits()) {
2458 if (!E)
2459 continue;
2461 }
2462
2464 PreCondVars.restore(CGF);
2465}
2466
2470 return;
2471 llvm::DenseSet<const VarDecl *> SIMDLCVs;
2475 for (const Expr *C : LoopDirective->counters()) {
2476 SIMDLCVs.insert(
2478 }
2479 }
2480 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2481 auto CurPrivate = C->privates().begin();
2482 for (const Expr *E : C->varlist()) {
2484 const auto *PrivateVD =
2487
2489 bool IsRegistered =
2491 assert(IsRegistered && "linear var already registered as private");
2492
2493 (void)IsRegistered;
2494 } else {
2496 }
2497 ++CurPrivate;
2498 }
2499 }
2500}
2501
2505 return;
2508 true);
2511
2512
2513
2515 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
2517 true);
2520
2521
2522
2524 }
2525}
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2577 return true;
2578
2579 if (const auto *CS = dyn_cast(S))
2581
2584 return true;
2585 }
2586
2587 return false;
2588}
2589
2592
2593
2595
2596
2597 if (HasOrderedDirective)
2599}
2600
2602
2603 LoopStack.setParallel(true);
2605 const Stmt *AssociatedStmt = D.getAssociatedStmt();
2608 if (const auto *C = D.getSingleClause<OMPOrderClause>())
2609 if (C->getKind() == OMPC_ORDER_concurrent)
2610 LoopStack.setParallel(true);
2612 if ((EKind == OMPD_simd ||
2616 return C->getModifier() == OMPC_REDUCTION_inscan;
2617 }))
2618
2619 LoopStack.setParallel(false);
2620}
2621
2624 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2626 return;
2627 llvm::BasicBlock *DoneBB = nullptr;
2628 auto IC = D.counters().begin();
2629 auto IPC = D.private_counters().begin();
2630 for (const Expr *F : D.finals()) {
2633 const auto *CED = dyn_cast(OrigVD);
2634 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
2635 OrigVD->hasGlobalStorage() || CED) {
2636 if (!DoneBB) {
2637 if (llvm::Value *Cond = CondGen(*this)) {
2638
2639
2640 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
2642 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2644 }
2645 }
2647 if (CED) {
2649 } else {
2651 false,
2654 }
2656 VarScope.addPrivate(OrigVD, OrigAddr);
2659 }
2660 ++IC;
2661 ++IPC;
2662 }
2663 if (DoneBB)
2664 EmitBlock(DoneBB, true);
2665}
2666
2673
2674
2681
2685 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
2689 SimdInitGen(CGF);
2690
2691 BodyCodeGen(CGF);
2692 };
2696
2697 BodyCodeGen(CGF);
2698 };
2699 const Expr *IfCond = nullptr;
2702 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2704 (C->getNameModifier() == OMPD_unknown ||
2705 C->getNameModifier() == OMPD_simd)) {
2706 IfCond = C->getCondition();
2707 break;
2708 }
2709 }
2710 }
2711 if (IfCond) {
2713 } else {
2715 ThenRCG(CGF);
2716 }
2717}
2718
2721 Action.Enter(CGF);
2722 OMPLoopScope PreInitScope(CGF, S);
2723
2724
2725
2726
2727
2728
2729
2730
2737 }
2738
2740
2741
2742
2743 bool CondConstant;
2744 llvm::BasicBlock *ContBlock = nullptr;
2746 if (!CondConstant)
2747 return;
2748 } else {
2749 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
2751 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2755 }
2756
2757
2758 const Expr *IVExpr = S.getIterationVariable();
2762
2763
2764
2765
2766 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) {
2768
2770 }
2771
2774 {
2781 CGF, S, CGF.EmitLValue(S.getIterationVariable()));
2786
2788 CGF, S,
2791 },
2796 emitOMPLoopBodyWithStopPoint(CGF, S,
2797 CodeGenFunction::JumpDest());
2798 },
2800 });
2802
2803 if (HasLastprivateClause)
2810 }
2811
2812 if (ContBlock) {
2815 }
2816}
2817
2818
2819
2821
2823
2826 return false;
2827 }
2828
2829
2830
2831
2832 if (const auto *CanonLoop = dyn_cast(S.getRawStmt())) {
2833 if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2834 for (const Stmt *SubStmt : SyntacticalLoop->children()) {
2835 if (!SubStmt)
2836 continue;
2837 if (const CompoundStmt *CS = dyn_cast(SubStmt)) {
2838 for (const Stmt *CSSubStmt : CS->children()) {
2839 if (!CSSubStmt)
2840 continue;
2842 return false;
2843 }
2844 }
2845 }
2846 }
2847 }
2848 }
2849 return true;
2850}
2851
2852static llvm::MapVector<llvm::Value *, llvm::Value *>
2854 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
2855 for (const auto *Clause : S.getClausesOfKind<OMPAlignedClause>()) {
2856 llvm::APInt ClauseAlignment(64, 0);
2857 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2858 auto *AlignmentCI =
2860 ClauseAlignment = AlignmentCI->getValue();
2861 }
2862 for (const Expr *E : Clause->varlist()) {
2863 llvm::APInt Alignment(ClauseAlignment);
2864 if (Alignment == 0) {
2865
2866
2867
2868 Alignment =
2871 E->getType()->getPointeeType()))
2873 }
2874 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2875 "alignment is not power of 2");
2877 AlignedVars[PtrValue] = CGF.Builder.getInt64(Alignment.getSExtValue());
2878 }
2879 }
2880 return AlignedVars;
2881}
2882
2883
2884
2887 bool UseOMPIRBuilder =
2889 if (UseOMPIRBuilder) {
2890 auto &&CodeGenIRBuilder = [&S, &CGM, UseOMPIRBuilder](CodeGenFunction &CGF,
2892
2893 if (UseOMPIRBuilder) {
2894 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
2896
2897 const Stmt *Inner = S.getRawStmt();
2898 llvm::CanonicalLoopInfo *CLI =
2899 CGF.EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
2900
2901 llvm::OpenMPIRBuilder &OMPBuilder =
2903
2904 llvm::ConstantInt *Simdlen = nullptr;
2907 true);
2909 Simdlen = Val;
2910 }
2911 llvm::ConstantInt *Safelen = nullptr;
2914 true);
2916 Safelen = Val;
2917 }
2918 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
2919 if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
2920 if (C->getKind() == OpenMPOrderClauseKind::OMPC_ORDER_concurrent) {
2921 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
2922 }
2923 }
2924
2925
2926 OMPBuilder.applySimd(CLI, AlignedVars,
2927 nullptr, Order, Simdlen, Safelen);
2928 return;
2929 }
2930 };
2931 {
2932 auto LPCRegion =
2934 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
2936 CodeGenIRBuilder);
2937 }
2938 return;
2939 }
2940
2945 };
2946 {
2947 auto LPCRegion =
2949 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
2951 }
2952
2954}
2955
2959
2961
2962 OMPTransformDirectiveScopeRAII TileScope(*this, &S);
2964}
2965
2967
2968 OMPTransformDirectiveScopeRAII StripeScope(*this, &S);
2970}
2971
2973
2974 OMPTransformDirectiveScopeRAII ReverseScope(*this, &S);
2976}
2977
2980
2981 OMPTransformDirectiveScopeRAII InterchangeScope(*this, &S);
2983}
2984
2986
2987 OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
2989}
2990
2992 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
2993
2994 if (UseOMPIRBuilder) {
2996 const Stmt *Inner = S.getRawStmt();
2997
2998
2999
3000
3003
3004 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
3005
3007 llvm::CanonicalLoopInfo *UnrolledCLI = nullptr;
3008
3011 OMPBuilder.unrollLoopFull(DL, CLI);
3012 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
3013 uint64_t Factor = 0;
3014 if (Expr *FactorExpr = PartialClause->getFactor()) {
3015 Factor = FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
3016 assert(Factor >= 1 && "Only positive factors are valid");
3017 }
3018 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
3019 NeedsUnrolledCLI ? &UnrolledCLI : nullptr);
3020 } else {
3021 OMPBuilder.unrollLoopHeuristic(DL, CLI);
3022 }
3023
3024 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
3025 "NeedsUnrolledCLI implies UnrolledCLI to be set");
3026 if (UnrolledCLI)
3028
3029 return;
3030 }
3031
3032
3033
3034
3035
3036
3038
3041 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
3042 if (Expr *FactorExpr = PartialClause->getFactor()) {
3043 uint64_t Factor =
3044 FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
3045 assert(Factor >= 1 && "Only positive factors are valid");
3046 LoopStack.setUnrollCount(Factor);
3047 }
3048 }
3049
3050 EmitStmt(S.getAssociatedStmt());
3051}
3052
3053void CodeGenFunction::EmitOMPOuterLoop(
3054 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
3056 const CodeGenFunction::OMPLoopArguments &LoopArgs,
3060
3061 const Expr *IVExpr = S.getIterationVariable();
3064
3066
3067
3068 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
3070 const SourceRange R = S.getSourceRange();
3074
3075 llvm::Value *BoolCondVal = nullptr;
3076 if (!DynamicOrOrdered) {
3077
3078
3079
3081
3083
3085 } else {
3086 BoolCondVal =
3087 RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
3088 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
3089 }
3090
3091
3092
3093 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
3096
3097 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
3098 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
3099 if (ExitBlock != LoopExit.getBlock()) {
3102 }
3104
3105
3106
3107 if (DynamicOrOrdered)
3109
3110
3112 BreakContinueStack.push_back(BreakContinue(S, LoopExit, Continue));
3113
3116 *this, S,
3117 [&S, IsMonotonic, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
3118
3119
3122 if (const auto *C = S.getSingleClause())
3123 if (C->getKind() == OMPC_ORDER_concurrent)
3125 } else {
3127 }
3128 },
3129 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
3130 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3131 SourceLocation Loc = S.getBeginLoc();
3132
3133
3134
3135
3136
3137 CGF.EmitOMPInnerLoop(
3138 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
3139 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3140 CodeGenLoop(CGF, S, LoopExit);
3141 },
3142 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
3143 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
3144 });
3145 });
3146
3148 BreakContinueStack.pop_back();
3149 if (!DynamicOrOrdered) {
3150
3153 }
3154
3158
3160
3161
3162 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {
3163 if (!DynamicOrOrdered)
3164 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3165 LoopArgs.DKind);
3166 };
3167 OMPCancelStack.emitExit(*this, EKind, CodeGen);
3168}
3169
3170void CodeGenFunction::EmitOMPForOuterLoop(
3171 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
3172 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
3173 const OMPLoopArguments &LoopArgs,
3175 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3176
3177
3178 const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind.Schedule);
3179
3181 LoopArgs.Chunk != nullptr)) &&
3182 "static non-chunked schedule does not need outer loop");
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236 const Expr *IVExpr = S.getIterationVariable();
3239
3240 if (DynamicOrOrdered) {
3241 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
3242 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
3243 llvm::Value *LBVal = DispatchBounds.first;
3244 llvm::Value *UBVal = DispatchBounds.second;
3245 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
3246 LoopArgs.Chunk};
3248 IVSigned, Ordered, DipatchRTInputValues);
3249 } else {
3250 CGOpenMPRuntime::StaticRTInput StaticInit(
3251 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3252 LoopArgs.ST, LoopArgs.Chunk);
3254 RT.emitForStaticInit(*this, S.getBeginLoc(), EKind, ScheduleKind,
3255 StaticInit);
3256 }
3257
3258 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3259 const unsigned IVSize,
3260 const bool IVSigned) {
3261 if (Ordered) {
3263 IVSigned);
3264 }
3265 };
3266
3267 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3268 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3269 OuterLoopArgs.IncExpr = S.getInc();
3270 OuterLoopArgs.Init = S.getInit();
3271 OuterLoopArgs.Cond = S.getCond();
3272 OuterLoopArgs.NextLB = S.getNextLowerBound();
3273 OuterLoopArgs.NextUB = S.getNextUpperBound();
3274 OuterLoopArgs.DKind = LoopArgs.DKind;
3275 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3277 if (DynamicOrOrdered) {
3279 }
3280}
3281
3283 const unsigned IVSize, const bool IVSigned) {}
3284
3285void CodeGenFunction::EmitOMPDistributeOuterLoop(
3287 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
3289
3290 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3291
3292
3293
3294
3295
3296
3297 const Expr *IVExpr = S.getIterationVariable();
3301
3302 CGOpenMPRuntime::StaticRTInput StaticInit(
3303 IVSize, IVSigned, false, LoopArgs.IL, LoopArgs.LB,
3304 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3306
3307
3308
3309 Expr *IncExpr;
3311 IncExpr = S.getDistInc();
3312 else
3313 IncExpr = S.getInc();
3314
3315
3316
3317
3318 OMPLoopArguments OuterLoopArgs;
3319 OuterLoopArgs.LB = LoopArgs.LB;
3320 OuterLoopArgs.UB = LoopArgs.UB;
3321 OuterLoopArgs.ST = LoopArgs.ST;
3322 OuterLoopArgs.IL = LoopArgs.IL;
3323 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3325 ? S.getCombinedEnsureUpperBound()
3326 : S.getEnsureUpperBound();
3327 OuterLoopArgs.IncExpr = IncExpr;
3329 ? S.getCombinedInit()
3330 : S.getInit();
3332 ? S.getCombinedCond()
3333 : S.getCond();
3335 ? S.getCombinedNextLowerBound()
3336 : S.getNextLowerBound();
3338 ? S.getCombinedNextUpperBound()
3339 : S.getNextUpperBound();
3340 OuterLoopArgs.DKind = OMPD_distribute;
3341
3342 EmitOMPOuterLoop( false, false, S,
3343 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3345}
3346
3347static std::pair<LValue, LValue>
3351 LValue LB =
3353 LValue UB =
3355
3356
3357
3358
3359
3360
3361
3362 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
3363 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
3365 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
3367 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
3368 LS.getIterationVariable()->getType(),
3369 LS.getPrevLowerBoundVariable()->getExprLoc());
3371 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
3373 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
3374 LS.getIterationVariable()->getType(),
3375 LS.getPrevUpperBoundVariable()->getExprLoc());
3376
3379
3380 return {LB, UB};
3381}
3382
3383
3384
3385
3386
3387
3388
3389
3390static std::pair<llvm::Value *, llvm::Value *>
3395 const Expr *IVExpr = LS.getIterationVariable();
3396
3397
3398
3399
3401 llvm::Value *LBVal =
3402 CGF.EmitLoadOfScalar(LB, false, IteratorTy, S.getBeginLoc());
3403 llvm::Value *UBVal =
3404 CGF.EmitLoadOfScalar(UB, false, IteratorTy, S.getBeginLoc());
3405 return {LBVal, UBVal};
3406}
3407
3412 LValue LB =
3414 llvm::Value *LBCast = CGF.Builder.CreateIntCast(
3416 CapturedVars.push_back(LBCast);
3417 LValue UB =
3419
3420 llvm::Value *UBCast = CGF.Builder.CreateIntCast(
3422 CapturedVars.push_back(UBCast);
3423}
3424
3425static void
3430 auto &&CGInlinedWorksharingLoop = [&S, EKind](CodeGenFunction &CGF,
3432 Action.Enter(CGF);
3433 bool HasCancel = false;
3435 if (const auto *D = dyn_cast(&S))
3436 HasCancel = D->hasCancel();
3437 else if (const auto *D = dyn_cast(&S))
3438 HasCancel = D->hasCancel();
3439 else if (const auto *D =
3440 dyn_cast(&S))
3441 HasCancel = D->hasCancel();
3442 }
3447 };
3448
3451 CGInlinedWorksharingLoop,
3453}
3454
3459 S.getDistInc());
3460 };
3461 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3462 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3463}
3464
3469 S.getDistInc());
3470 };
3471 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3472 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3473}
3474
3479 };
3480 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3481 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
3482}
3483
3486
3489 };
3490 llvm::Function *Fn;
3491 llvm::Constant *Addr;
3492
3493 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3494 S, ParentName, Fn, Addr, true, CodeGen);
3495 assert(Fn && Addr && "Target device function emission failed.");
3496}
3497
3505
3506namespace {
3507struct ScheduleKindModifiersTy {
3514 : Kind(Kind), M1(M1), M2(M2) {}
3515};
3516}
3517
3522
3523 const auto *IVExpr = cast(S.getIterationVariable());
3524 const auto *IVDecl = cast(IVExpr->getDecl());
3526
3527
3528
3529
3530 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) {
3532
3534 }
3535
3537
3538 bool HasLastprivateClause;
3539
3540 {
3541 OMPLoopScope PreInitScope(*this, S);
3542
3543
3544
3545 bool CondConstant;
3546 llvm::BasicBlock *ContBlock = nullptr;
3548 if (!CondConstant)
3549 return false;
3550 } else {
3551 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3553 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3557 }
3558
3560 bool Ordered = false;
3561 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
3562 if (OrderedClause->getNumForLoops())
3563 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
3564 else
3565 Ordered = true;
3566 }
3567
3570
3571
3572 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
3573 LValue LB = Bounds.first;
3574 LValue UB = Bounds.second;
3579
3580
3581 {
3585
3586
3587
3588 CGM.getOpenMPRuntime().emitBarrierCall(
3589 *this, S.getBeginLoc(), OMPD_unknown, false,
3590 true);
3591 }
3594 *this, S, EmitLValue(S.getIterationVariable()));
3601 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
3602
3603
3604 const Expr *ChunkExpr = nullptr;
3607 ScheduleKind.Schedule = C->getScheduleKind();
3608 ScheduleKind.M1 = C->getFirstScheduleModifier();
3609 ScheduleKind.M2 = C->getSecondScheduleModifier();
3610 ChunkExpr = C->getChunkSize();
3611 } else {
3612
3613 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
3614 *this, S, ScheduleKind.Schedule, ChunkExpr);
3615 }
3616 bool HasChunkSizeOne = false;
3617 llvm::Value *Chunk = nullptr;
3618 if (ChunkExpr) {
3621 S.getIterationVariable()->getType(),
3622 S.getBeginLoc());
3625 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
3626 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3627 }
3628 }
3631
3632
3633
3634
3635 bool StaticChunkedOne =
3637 Chunk != nullptr) &&
3639 bool IsMonotonic =
3640 Ordered ||
3641 (ScheduleKind.Schedule == OMPC_SCHEDULE_static &&
3642 !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3643 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3644 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3645 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3647 Chunk != nullptr) ||
3648 StaticChunkedOne) &&
3649 !Ordered) {
3653 *this, S,
3657 } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
3658 if (C->getKind() == OMPC_ORDER_concurrent)
3660 }
3661 },
3662 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3663 &S, ScheduleKind, LoopExit, EKind,
3665
3666
3667
3668
3669
3671 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
3672 UB.getAddress(), ST.getAddress(),
3673 StaticChunkedOne ? Chunk : nullptr);
3675 CGF, S.getBeginLoc(), EKind, ScheduleKind, StaticInit);
3676
3677 if (!StaticChunkedOne)
3679
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3696 StaticChunkedOne ? S.getCombinedParForInDistCond()
3697 : S.getCond(),
3698 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3699 [&S, LoopExit](CodeGenFunction &CGF) {
3700 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3701 },
3702 [](CodeGenFunction &) {});
3703 });
3705
3706 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
3708 OMPD_for);
3709 };
3710 OMPCancelStack.emitExit(*this, EKind, CodeGen);
3711 } else {
3712
3713
3716 EUB);
3717 LoopArguments.DKind = OMPD_for;
3718 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3719 LoopArguments, CGDispatchBounds);
3720 }
3723 return CGF.Builder.CreateIsNotNull(
3725 });
3726 }
3729 ? OMPD_parallel_for_simd
3730 : OMPD_parallel);
3731
3733 *this, S, [IL, &S](CodeGenFunction &CGF) {
3734 return CGF.Builder.CreateIsNotNull(
3736 });
3737
3738 if (HasLastprivateClause)
3744 return CGF.Builder.CreateIsNotNull(
3746 });
3747 }
3749
3750 if (ContBlock) {
3752 EmitBlock(ContBlock, true);
3753 }
3754 }
3755 return HasLastprivateClause;
3756}
3757
3758
3759
3760
3761static std::pair<LValue, LValue>
3764 LValue LB =
3766 LValue UB =
3768 return {LB, UB};
3769}
3770
3771
3772
3773
3774
3775static std::pair<llvm::Value *, llvm::Value *>
3779 const Expr *IVExpr = LS.getIterationVariable();
3781 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
3782 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
3783 return {LBVal, UBVal};
3784}
3785
3786
3787
3788
3789
3790
3791
3792
3795 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
3796 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3797 NumIteratorsGen(CGF), CGF.SizeTy, false);
3803 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3804 "Only inscan reductions are expected.");
3805 Shareds.append(C->varlist_begin(), C->varlist_end());
3806 Privates.append(C->privates().begin(), C->privates().end());
3807 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3808 CopyArrayTemps.append(C->copy_array_temps().begin(),
3809 C->copy_array_temps().end());
3810 }
3811 {
3812
3813
3814
3815 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
3816 unsigned Count = 0;
3817 auto *ITA = CopyArrayTemps.begin();
3818 for (const Expr *IRef : Privates) {
3820
3821
3822 if (PrivateVD->getType()->isVariablyModifiedType()) {
3825 }
3827 CGF,
3830 ->getSizeExpr()),
3832
3834 ++ITA;
3835 ++Count;
3836 }
3837 }
3838}
3839
3840
3841
3842
3843
3844
3847 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
3848 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3849 NumIteratorsGen(CGF), CGF.SizeTy, false);
3857 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3858 "Only inscan reductions are expected.");
3859 Shareds.append(C->varlist_begin(), C->varlist_end());
3860 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3861 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3862 Privates.append(C->privates().begin(), C->privates().end());
3863 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
3864 CopyArrayElems.append(C->copy_array_elems().begin(),
3865 C->copy_array_elems().end());
3866 }
3867
3868
3869 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
3870 OMPScanNumIterations,
3871 llvm::ConstantInt::get(CGF.SizeTy, 1, false));
3872 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
3873 const Expr *PrivateExpr = Privates[I];
3874 const Expr *OrigExpr = Shareds[I];
3875 const Expr *CopyArrayElem = CopyArrayElems[I];
3877 CGF,
3881 LValue DestLVal = CGF.EmitLValue(OrigExpr);
3882 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem);
3884 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
3887 }
3888}
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3911 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen,
3913 llvm::function_ref<void(CodeGenFunction &)> SecondGen) {
3914 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3915 NumIteratorsGen(CGF), CGF.SizeTy, false);
3922 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3923 "Only inscan reductions are expected.");
3924 Privates.append(C->privates().begin(), C->privates().end());
3925 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3926 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3927 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3928 CopyArrayElems.append(C->copy_array_elems().begin(),
3929 C->copy_array_elems().end());
3930 }
3932 {
3933
3934
3935
3936
3937
3938
3941 FirstGen(CGF);
3942 }
3943
3944 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3945 &ReductionOps,
3947 Action.Enter(CGF);
3948
3949
3950
3951 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3952 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body");
3953 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit");
3954 llvm::Function *F =
3955 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3956 llvm::Value *Arg =
3957 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3958 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3959 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3960 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3961 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3962 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3963 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3965 CGF.EmitBlock(LoopBB);
3966 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3967
3968 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3969 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3970 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3971
3972
3973 llvm::BasicBlock *InnerLoopBB =
3974 CGF.createBasicBlock("omp.inner.log.scan.body");
3975 llvm::BasicBlock *InnerExitBB =
3976 CGF.createBasicBlock("omp.inner.log.scan.exit");
3977 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3978 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3979 CGF.EmitBlock(InnerLoopBB);
3980 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3981 IVal->addIncoming(NMin1, LoopBB);
3982 {
3984 auto *ILHS = LHSs.begin();
3985 auto *IRHS = RHSs.begin();
3986 for (const Expr *CopyArrayElem : CopyArrayElems) {
3990 {
3992 CGF,
3996 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
3997 }
3998 PrivScope.addPrivate(LHSVD, LHSAddr);
4000 {
4001 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
4003 CGF,
4007 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
4008 }
4009 PrivScope.addPrivate(RHSVD, RHSAddr);
4010 ++ILHS;
4011 ++IRHS;
4012 }
4014 CGF.CGM.getOpenMPRuntime().emitReduction(
4015 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
4016 {true, true,
4017 {}, OMPD_unknown});
4018 }
4019 llvm::Value *NextIVal =
4020 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
4021 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
4022 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
4023 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
4024 CGF.EmitBlock(InnerExitBB);
4025 llvm::Value *Next =
4026 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
4027 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
4028
4029 llvm::Value *NextPow2K =
4030 CGF.Builder.CreateShl(Pow2K, 1, "", true);
4031 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
4032 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
4033 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
4035 CGF.EmitBlock(ExitBB);
4036 };
4041 CGF, S.getBeginLoc(), OMPD_unknown, false,
4042 true);
4043 } else {
4044 RegionCodeGenTy RCG(CodeGen);
4045 RCG(CGF);
4046 }
4047
4049 SecondGen(CGF);
4050}
4051
4054 bool HasCancel) {
4055 bool HasLastprivates;
4059 return C->getModifier() == OMPC_REDUCTION_inscan;
4060 })) {
4061 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4063 OMPLoopScope LoopScope(CGF, S);
4065 };
4066 const auto &&FirstGen = [&S, HasCancel, EKind](CodeGenFunction &CGF) {
4071
4073 OMPD_for);
4074 };
4075 const auto &&SecondGen = [&S, HasCancel, EKind,
4081 };
4087 } else {
4092 }
4093 return HasLastprivates;
4094}
4095
4096
4097
4099 bool HasCancel) {
4100 if (HasCancel)
4101 return false;
4104 continue;
4105
4106 if (auto *SC = dyn_cast(C)) {
4108 return false;
4110 return false;
4111 switch (SC->getScheduleKind()) {
4112 case OMPC_SCHEDULE_auto:
4113 case OMPC_SCHEDULE_dynamic:
4114 case OMPC_SCHEDULE_runtime:
4115 case OMPC_SCHEDULE_guided:
4116 case OMPC_SCHEDULE_static:
4117 continue;
4119 return false;
4120 }
4121 }
4122
4123 return false;
4124 }
4125
4126 return true;
4127}
4128
4129static llvm::omp::ScheduleKind
4131 switch (ScheduleClauseKind) {
4133 return llvm::omp::OMP_SCHEDULE_Default;
4134 case OMPC_SCHEDULE_auto:
4135 return llvm::omp::OMP_SCHEDULE_Auto;
4136 case OMPC_SCHEDULE_dynamic:
4137 return llvm::omp::OMP_SCHEDULE_Dynamic;
4138 case OMPC_SCHEDULE_guided:
4139 return llvm::omp::OMP_SCHEDULE_Guided;
4140 case OMPC_SCHEDULE_runtime:
4141 return llvm::omp::OMP_SCHEDULE_Runtime;
4142 case OMPC_SCHEDULE_static:
4143 return llvm::omp::OMP_SCHEDULE_Static;
4144 }
4145 llvm_unreachable("Unhandled schedule kind");
4146}
4147
4148
4149
4152 bool HasLastprivates = false;
4153 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder &&
4155 auto &&CodeGen = [&S, &CGM, HasCancel, &HasLastprivates,
4157
4158 if (UseOMPIRBuilder) {
4159 bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
4160
4161 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
4162 llvm::Value *ChunkSize = nullptr;
4163 if (auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {
4164 SchedKind =
4166 if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
4168 }
4169
4170
4171 const Stmt *Inner = S.getRawStmt();
4172 llvm::CanonicalLoopInfo *CLI =
4174
4175 llvm::OpenMPIRBuilder &OMPBuilder =
4177 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4179 cantFail(OMPBuilder.applyWorkshareLoop(
4180 CGF.Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
4181 SchedKind, ChunkSize, false,
4182 false, false,
4183 false));
4184 return;
4185 }
4186
4188 };
4189 {
4190 auto LPCRegion =
4192 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
4194 HasCancel);
4195 }
4196
4197 if (!UseOMPIRBuilder) {
4198
4199 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4201 }
4202
4204}
4205
4209
4211 bool HasLastprivates = false;
4212 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
4215 };
4216 {
4217 auto LPCRegion =
4219 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4220 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
4221 }
4222
4223
4224 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4225 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
4226
4228}
4229
4231 const Twine &Name,
4232 llvm::Value *Init = nullptr) {
4236 return LVal;
4237}
4238
4239void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
4240 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4241 const auto *CS = dyn_cast(CapturedStmt);
4242 bool HasLastprivates = false;
4244 auto &&CodeGen = [&S, CapturedStmt, CS, EKind,
4245 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
4246 const ASTContext &C = CGF.getContext();
4247 QualType KmpInt32Ty =
4248 C.getIntTypeForBitwidth(32, 1);
4249
4250 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
4251 CGF.Builder.getInt32(0));
4252 llvm::ConstantInt *GlobalUBVal = CS != nullptr
4253 ? CGF.Builder.getInt32(CS->size() - 1)
4254 : CGF.Builder.getInt32(0);
4255 LValue UB =
4256 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
4257 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
4258 CGF.Builder.getInt32(1));
4259 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
4260 CGF.Builder.getInt32(0));
4261
4262 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
4263 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4264 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4265 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4266 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4267
4270 S.getBeginLoc(), FPOptionsOverride());
4271
4274 S.getBeginLoc(), true, FPOptionsOverride());
4275 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
4288 llvm::SwitchInst *SwitchStmt =
4289 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4290 ExitBB, CS == nullptr ? 1 : CS->size());
4291 if (CS) {
4292 unsigned CaseNumber = 0;
4293 for (const Stmt *SubStmt : CS->children()) {
4294 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
4295 CGF.EmitBlock(CaseBB);
4296 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4297 CGF.EmitStmt(SubStmt);
4298 CGF.EmitBranch(ExitBB);
4299 ++CaseNumber;
4300 }
4301 } else {
4302 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
4303 CGF.EmitBlock(CaseBB);
4304 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4305 CGF.EmitStmt(CapturedStmt);
4306 CGF.EmitBranch(ExitBB);
4307 }
4308 CGF.EmitBlock(ExitBB, true);
4309 };
4310
4311 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4312 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4313
4314
4315
4316 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4317 CGF, S.getBeginLoc(), OMPD_unknown, false,
4318 true);
4319 }
4320 CGF.EmitOMPPrivateClause(S, LoopScope);
4321 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4322 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4323 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4324 (void)LoopScope.Privatize();
4326 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4327
4328
4329 OpenMPScheduleTy ScheduleKind;
4330 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
4331 CGOpenMPRuntime::StaticRTInput StaticInit(
4332 32, true, false, IL.getAddress(),
4333 LB.getAddress(), UB.getAddress(), ST.getAddress());
4334 CGF.CGM.getOpenMPRuntime().emitForStaticInit(CGF, S.getBeginLoc(), EKind,
4335 ScheduleKind, StaticInit);
4336
4337 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4338 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4339 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4340 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4341
4342 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4343
4344 CGF.EmitOMPInnerLoop(S, false, Cond, Inc, BodyGen,
4345 [](CodeGenFunction &) {});
4346
4347 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4348 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4349 OMPD_sections);
4350 };
4351 CGF.OMPCancelStack.emitExit(CGF, EKind, CodeGen);
4352 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
4353
4355 return CGF.Builder.CreateIsNotNull(
4357 });
4358
4359
4360 if (HasLastprivates)
4362 S, false,
4363 CGF.Builder.CreateIsNotNull(
4365 };
4366
4367 bool HasCancel = false;
4368 if (auto *OSD = dyn_cast(&S))
4369 HasCancel = OSD->hasCancel();
4370 else if (auto *OPSD = dyn_cast(&S))
4371 HasCancel = OPSD->hasCancel();
4373 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
4374 HasCancel);
4375
4376
4377
4378 if (HasLastprivates && S.getSingleClause()) {
4379
4380
4381 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4382 OMPD_unknown);
4383 }
4384}
4385
4387 {
4388
4390 Action.Enter(CGF);
4396 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4398 };
4399 auto LPCRegion =
4401 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4402 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_scope, CodeGen);
4403 }
4404
4406 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_scope);
4407 }
4408
4410}
4411
4413 if (CGM.getLangOpts().OpenMPIRBuilder) {
4414 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4415 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4416 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4417
4418 auto FiniCB = [](InsertPointTy IP) {
4419
4420
4421 return llvm::Error::success();
4422 };
4423
4424 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4426 const auto *CS = dyn_cast(CapturedStmt);
4428 if (CS) {
4429 for (const Stmt *SubStmt : CS->children()) {
4430 auto SectionCB = [this, SubStmt](InsertPointTy AllocaIP,
4431 InsertPointTy CodeGenIP) {
4433 *this, SubStmt, AllocaIP, CodeGenIP, "section");
4434 return llvm::Error::success();
4435 };
4436 SectionCBVector.push_back(SectionCB);
4437 }
4438 } else {
4439 auto SectionCB = [this, CapturedStmt](InsertPointTy AllocaIP,
4440 InsertPointTy CodeGenIP) {
4442 *this, CapturedStmt, AllocaIP, CodeGenIP, "section");
4443 return llvm::Error::success();
4444 };
4445 SectionCBVector.push_back(SectionCB);
4446 }
4447
4448
4449
4450
4451
4452 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4453 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4454
4455
4456 ReplVal = &Val;
4457
4458 return CodeGenIP;
4459 };
4460
4463 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4465 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4466 cantFail(OMPBuilder.createSections(
4467 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4469 Builder.restoreIP(AfterIP);
4470 return;
4471 }
4472 {
4473 auto LPCRegion =
4475 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4476 EmitSections(S);
4477 }
4478
4480 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4481 OMPD_sections);
4482 }
4483
4485}
4486
4488 if (CGM.getLangOpts().OpenMPIRBuilder) {
4489 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4490 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4491
4492 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4493 auto FiniCB = [this](InsertPointTy IP) {
4495 return llvm::Error::success();
4496 };
4497
4498 auto BodyGenCB = [SectionRegionBodyStmt, this](InsertPointTy AllocaIP,
4499 InsertPointTy CodeGenIP) {
4501 *this, SectionRegionBodyStmt, AllocaIP, CodeGenIP, "section");
4502 return llvm::Error::success();
4503 };
4504
4507 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4508 cantFail(OMPBuilder.createSection(Builder, BodyGenCB, FiniCB));
4509 Builder.restoreIP(AfterIP);
4510
4511 return;
4512 }
4515 EmitStmt(S.getAssociatedStmt());
4516}
4517
4523
4524
4525
4526
4528 CopyprivateVars.append(C->varlist_begin(), C->varlist_end());
4529 DestExprs.append(C->destination_exprs().begin(),
4530 C->destination_exprs().end());
4531 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
4532 AssignmentOps.append(C->assignment_ops().begin(),
4533 C->assignment_ops().end());
4534 }
4535
4537 Action.Enter(CGF);
4542 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4543 };
4544 {
4545 auto LPCRegion =
4547 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4548 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
4549 CopyprivateVars, DestExprs,
4550 SrcExprs, AssignmentOps);
4551 }
4552
4553
4554 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
4555 CGM.getOpenMPRuntime().emitBarrierCall(
4556 *this, S.getBeginLoc(),
4557 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
4558 }
4559
4561}
4562
4565 Action.Enter(CGF);
4566 CGF.EmitStmt(S.getRawStmt());
4567 };
4569}
4570
4572 if (CGM.getLangOpts().OpenMPIRBuilder) {
4573 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4574 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4575
4576 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4577
4578 auto FiniCB = [this](InsertPointTy IP) {
4580 return llvm::Error::success();
4581 };
4582
4583 auto BodyGenCB = [MasterRegionBodyStmt, this](InsertPointTy AllocaIP,
4584 InsertPointTy CodeGenIP) {
4586 *this, MasterRegionBodyStmt, AllocaIP, CodeGenIP, "master");
4587 return llvm::Error::success();
4588 };
4589
4592 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4593 cantFail(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
4594 Builder.restoreIP(AfterIP);
4595
4596 return;
4597 }
4601}
4602
4605 Action.Enter(CGF);
4606 CGF.EmitStmt(S.getRawStmt());
4607 };
4608 Expr *Filter = nullptr;
4609 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4610 Filter = FilterClause->getThreadID();
4612 Filter);
4613}
4614
4616 if (CGM.getLangOpts().OpenMPIRBuilder) {
4617 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4618 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4619
4620 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4621 const Expr *Filter = nullptr;
4622 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4623 Filter = FilterClause->getThreadID();
4624 llvm::Value *FilterVal = Filter
4626 : llvm::ConstantInt::get(CGM.Int32Ty, 0);
4627
4628 auto FiniCB = [this](InsertPointTy IP) {
4630 return llvm::Error::success();
4631 };
4632
4633 auto BodyGenCB = [MaskedRegionBodyStmt, this](InsertPointTy AllocaIP,
4634 InsertPointTy CodeGenIP) {
4636 *this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP, "masked");
4637 return llvm::Error::success();
4638 };
4639
4642 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
4643 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal));
4644 Builder.restoreIP(AfterIP);
4645
4646 return;
4647 }
4651}
4652
4654 if (CGM.getLangOpts().OpenMPIRBuilder) {
4655 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4656 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4657
4658 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4659 const Expr *Hint = nullptr;
4660 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4661 Hint = HintClause->getHint();
4662
4663
4664
4665
4666 llvm::Value *HintInst = nullptr;
4667 if (Hint)
4668 HintInst =
4670
4671 auto FiniCB = [this](InsertPointTy IP) {
4673 return llvm::Error::success();
4674 };
4675
4676 auto BodyGenCB = [CriticalRegionBodyStmt, this](InsertPointTy AllocaIP,
4677 InsertPointTy CodeGenIP) {
4679 *this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP, "critical");
4680 return llvm::Error::success();
4681 };
4682
4685 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4686 cantFail(OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB,
4687 S.getDirectiveName().getAsString(),
4688 HintInst));
4689 Builder.restoreIP(AfterIP);
4690
4691 return;
4692 }
4693
4695 Action.Enter(CGF);
4696 CGF.EmitStmt(S.getAssociatedStmt());
4697 };
4698 const Expr *Hint = nullptr;
4699 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4700 Hint = HintClause->getHint();
4703 CGM.getOpenMPRuntime().emitCriticalRegion(*this,
4704 S.getDirectiveName().getAsString(),
4705 CodeGen, S.getBeginLoc(), Hint);
4706}
4707
4709 const OMPParallelForDirective &S) {
4710
4711
4713 Action.Enter(CGF);
4716 };
4717 {
4718 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4722 OMPLoopScope LoopScope(CGF, S);
4724 };
4725 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4727 return C->getModifier() == OMPC_REDUCTION_inscan;
4728 });
4729 if (IsInscan)
4731 auto LPCRegion =
4735 if (IsInscan)
4737 }
4738
4740}
4741
4743 const OMPParallelForSimdDirective &S) {
4744
4745
4747 Action.Enter(CGF);
4750 };
4751 {
4752 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4756 OMPLoopScope LoopScope(CGF, S);
4758 };
4759 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4761 return C->getModifier() == OMPC_REDUCTION_inscan;
4762 });
4763 if (IsInscan)
4765 auto LPCRegion =
4769 if (IsInscan)
4771 }
4772
4774}
4775
4777 const OMPParallelMasterDirective &S) {
4778
4779
4781 Action.Enter(CGF);
4790 };
4791 {
4792 auto LPCRegion =
4797 [](CodeGenFunction &) { return nullptr; });
4798 }
4799
4801}
4802
4804 const OMPParallelMaskedDirective &S) {
4805
4806
4808 Action.Enter(CGF);
4817 };
4818 {
4819 auto LPCRegion =
4824 [](CodeGenFunction &) { return nullptr; });
4825 }
4826
4828}
4829
4831 const OMPParallelSectionsDirective &S) {
4832
4833
4835 Action.Enter(CGF);
4837 CGF.EmitSections(S);
4838 };
4839 {
4840 auto LPCRegion =
4844 }
4845
4847}
4848
4849namespace {
4850
4851class CheckVarsEscapingUntiedTaskDeclContext final
4852 : public ConstStmtVisitor {
4854
4855public:
4856 explicit CheckVarsEscapingUntiedTaskDeclContext() = default;
4857 ~CheckVarsEscapingUntiedTaskDeclContext() = default;
4858 void VisitDeclStmt(const DeclStmt *S) {
4859 if (!S)
4860 return;
4861
4862 for (const Decl *D : S->decls()) {
4863 if (const auto *VD = dyn_cast_or_null(D))
4865 PrivateDecls.push_back(VD);
4866 }
4867 }
4869 void VisitCapturedStmt(const CapturedStmt *) {}
4870 void VisitLambdaExpr(const LambdaExpr *) {}
4871 void VisitBlockExpr(const BlockExpr *) {}
4872 void VisitStmt(const Stmt *S) {
4873 if (!S)
4874 return;
4875 for (const Stmt *Child : S->children())
4876 if (Child)
4877 Visit(Child);
4878 }
4879
4880
4881 ArrayRef<const VarDecl *> getPrivateDecls() const { return PrivateDecls; }
4882};
4883}
4884
4887
4888
4889 bool OmpAllMemory = false;
4890 if (llvm::any_of(
4892 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4893 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4894 })) {
4895 OmpAllMemory = true;
4896
4897
4898
4900 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
4901 nullptr);
4902
4903 DD.DepExprs.push_back(nullptr);
4904 }
4905
4906
4907 for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
4909 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
4910 continue;
4911 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
4912 continue;
4914 Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
4915 DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
4916 }
4917}
4918
4923
4924 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4926 auto PartId = std::next(I);
4927 auto TaskT = std::next(I, 4);
4928
4929 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
4930
4931
4932 const Expr *Cond = Clause->getCondition();
4933 bool CondConstant;
4935 Data.Final.setInt(CondConstant);
4936 else
4938 } else {
4939
4940 Data.Final.setInt(false);
4941 }
4942
4943 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
4944 const Expr *Prio = Clause->getPriority();
4945 Data.Priority.setInt(true);
4948 getContext().getIntTypeForBitwidth(32, 1),
4950 }
4951
4952
4953 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
4954
4956 auto IRef = C->varlist_begin();
4957 for (const Expr *IInit : C->private_copies()) {
4959 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4960 Data.PrivateVars.push_back(*IRef);
4961 Data.PrivateCopies.push_back(IInit);
4962 }
4963 ++IRef;
4964 }
4965 }
4966 EmittedAsPrivate.clear();
4967
4969 auto IRef = C->varlist_begin();
4970 auto IElemInitRef = C->inits().begin();
4971 for (const Expr *IInit : C->private_copies()) {
4973 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4974 Data.FirstprivateVars.push_back(*IRef);
4975 Data.FirstprivateCopies.push_back(IInit);
4976 Data.FirstprivateInits.push_back(*IElemInitRef);
4977 }
4978 ++IRef;
4979 ++IElemInitRef;
4980 }
4981 }
4982
4983 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4985 auto IRef = C->varlist_begin();
4986 auto ID = C->destination_exprs().begin();
4987 for (const Expr *IInit : C->private_copies()) {
4989 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4990 Data.LastprivateVars.push_back(*IRef);
4991 Data.LastprivateCopies.push_back(IInit);
4992 }
4993 LastprivateDstsOrigs.insert(
4996 ++IRef;
4997 ++ID;
4998 }
4999 }
5003 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
5004 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5005 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
5006 Data.ReductionOps.append(C->reduction_ops().begin(),
5007 C->reduction_ops().end());
5008 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5009 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5010 }
5011 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
5012 *this, S.getBeginLoc(), LHSs, RHSs, Data);
5013
5015
5016 if (.Tied) {
5017 CheckVarsEscapingUntiedTaskDeclContext Checker;
5018 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
5019 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
5020 Checker.getPrivateDecls().end());
5021 }
5022 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
5023 CapturedRegion](CodeGenFunction &CGF,
5025 llvm::MapVector<CanonicalDeclPtr,
5026 std::pair<Address, Address>>
5027 UntiedLocalVars;
5028
5030
5031 if (auto *DI = CGF.getDebugInfo()) {
5032 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
5033 CGF.CapturedStmtInfo->getCaptureFields();
5034 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
5035 if (CaptureFields.size() && ContextValue) {
5036 unsigned CharWidth = CGF.getContext().getCharWidth();
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050 for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
5051 const VarDecl *SharedVar = It->first;
5054 CGF.getContext().getASTRecordLayout(CaptureRecord);
5055 unsigned Offset =
5056 Layout.getFieldOffset(It->second->getFieldIndex()) / CharWidth;
5057 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5058 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
5059 CGF.Builder, false);
5060
5061
5062 auto UpdateExpr = [](llvm::LLVMContext &Ctx, auto *Declare,
5063 unsigned Offset) {
5065
5066 if (Offset) {
5067 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
5068 Ops.push_back(Offset);
5069 }
5070 Ops.push_back(llvm::dwarf::DW_OP_deref);
5071 Declare->setExpression(llvm::DIExpression::get(Ctx, Ops));
5072 };
5073 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
5074 if (auto DDI = dyn_castllvm::DbgVariableIntrinsic(&Last))
5075 UpdateExpr(DDI->getContext(), DDI, Offset);
5076
5077
5078 assert(.isTerminator() && "unexpected terminator");
5079 if (auto *Marker =
5080 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
5081 for (llvm::DbgVariableRecord &DVR : llvm::reverse(
5082 llvm::filterDbgVars(Marker->getDbgRecordRange()))) {
5083 UpdateExpr(Last.getContext(), &DVR, Offset);
5084 break;
5085 }
5086 }
5087 }
5088 }
5089 }
5091 if (.PrivateVars.empty() ||
.FirstprivateVars.empty() ||
5092 .LastprivateVars.empty() ||
.PrivateLocals.empty()) {
5093 enum { PrivatesParam = 2, CopyFnParam = 3 };
5094 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5096 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5098
5102 CallArgs.push_back(PrivatesPtr);
5103 ParamTypes.push_back(PrivatesPtr->getType());
5104 for (const Expr *E : Data.PrivateVars) {
5106 RawAddress PrivatePtr = CGF.CreateMemTemp(
5107 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
5108 PrivatePtrs.emplace_back(VD, PrivatePtr);
5109 CallArgs.push_back(PrivatePtr.getPointer());
5110 ParamTypes.push_back(PrivatePtr.getType());
5111 }
5112 for (const Expr *E : Data.FirstprivateVars) {
5115 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
5116 ".firstpriv.ptr.addr");
5117 PrivatePtrs.emplace_back(VD, PrivatePtr);
5118 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
5119 CallArgs.push_back(PrivatePtr.getPointer());
5120 ParamTypes.push_back(PrivatePtr.getType());
5121 }
5122 for (const Expr *E : Data.LastprivateVars) {
5125 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
5126 ".lastpriv.ptr.addr");
5127 PrivatePtrs.emplace_back(VD, PrivatePtr);
5128 CallArgs.push_back(PrivatePtr.getPointer());
5129 ParamTypes.push_back(PrivatePtr.getType());
5130 }
5131 for (const VarDecl *VD : Data.PrivateLocals) {
5134 Ty = CGF.getContext().getPointerType(Ty);
5136 Ty = CGF.getContext().getPointerType(Ty);
5137 RawAddress PrivatePtr = CGF.CreateMemTemp(
5138 CGF.getContext().getPointerType(Ty), ".local.ptr.addr");
5139 auto Result = UntiedLocalVars.insert(
5140 std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid())));
5141
5142 if (Result.second == false)
5143 *Result.first = std::make_pair(
5145 CallArgs.push_back(PrivatePtr.getPointer());
5146 ParamTypes.push_back(PrivatePtr.getType());
5147 }
5148 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5149 ParamTypes, false);
5150 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5151 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5152 for (const auto &Pair : LastprivateDstsOrigs) {
5153 const auto *OrigVD = cast(Pair.second->getDecl());
5155
5156 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
5158 Pair.second->getExprLoc());
5159 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress());
5160 }
5161 for (const auto &Pair : PrivatePtrs) {
5163 CGF.Builder.CreateLoad(Pair.second),
5164 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5165 CGF.getContext().getDeclAlign(Pair.first));
5166 Scope.addPrivate(Pair.first, Replacement);
5167 if (auto *DI = CGF.getDebugInfo())
5168 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5169 (void)DI->EmitDeclareOfAutoVariable(
5170 Pair.first, Pair.second.getBasePointer(), CGF.Builder,
5171 true);
5172 }
5173
5174
5175 for (auto &Pair : UntiedLocalVars) {
5176 QualType VDType = Pair.first->getType().getNonReferenceType();
5177 if (Pair.first->getType()->isLValueReferenceType())
5178 VDType = CGF.getContext().getPointerType(VDType);
5180 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5182 Ptr,
5183 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
5184 CGF.getPointerAlign());
5185 Pair.second.first = Replacement;
5186 Ptr = CGF.Builder.CreateLoad(Replacement);
5187 Replacement = Address(Ptr, CGF.ConvertTypeForMem(VDType),
5188 CGF.getContext().getDeclAlign(Pair.first));
5189 Pair.second.second = Replacement;
5190 } else {
5191 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5192 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
5193 CGF.getContext().getDeclAlign(Pair.first));
5194 Pair.second.first = Replacement;
5195 }
5196 }
5197 }
5198 if (Data.Reductions) {
5200 for (const auto &Pair : FirstprivatePtrs) {
5202 CGF.Builder.CreateLoad(Pair.second),
5203 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5204 CGF.getContext().getDeclAlign(Pair.first));
5205 FirstprivateScope.addPrivate(Pair.first, Replacement);
5206 }
5207 (void)FirstprivateScope.Privatize();
5208 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5210 Data.ReductionCopies, Data.ReductionOps);
5211 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5213 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5216
5217
5218
5219 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5220 RedCG, Cnt);
5221 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5222 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5224 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
5225 CGF.getContext().VoidPtrTy,
5226 CGF.getContext().getPointerType(
5227 Data.ReductionCopies[Cnt]->getType()),
5228 Data.ReductionCopies[Cnt]->getExprLoc()),
5229 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
5230 Replacement.getAlignment());
5233 }
5234 }
5235
5236 (void)Scope.Privatize();
5242 auto IPriv = C->privates().begin();
5243 auto IRed = C->reduction_ops().begin();
5244 auto ITD = C->taskgroup_descriptors().begin();
5245 for (const Expr *Ref : C->varlist()) {
5246 InRedVars.emplace_back(Ref);
5247 InRedPrivs.emplace_back(*IPriv);
5248 InRedOps.emplace_back(*IRed);
5249 TaskgroupDescriptors.emplace_back(*ITD);
5250 std::advance(IPriv, 1);
5251 std::advance(IRed, 1);
5252 std::advance(ITD, 1);
5253 }
5254 }
5255
5256
5258 if (!InRedVars.empty()) {
5259 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
5260 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5263
5264
5265
5266
5267
5268 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5269 RedCG, Cnt);
5270 llvm::Value *ReductionsPtr;
5271 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5272 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
5273 TRExpr->getExprLoc());
5274 } else {
5275 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5276 }
5277 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5278 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5280 CGF.EmitScalarConversion(
5281 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
5282 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5283 InRedPrivs[Cnt]->getExprLoc()),
5284 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5285 Replacement.getAlignment());
5288 }
5289 }
5291
5293 UntiedLocalVars);
5294 Action.Enter(CGF);
5295 BodyGen(CGF);
5296 };
5298 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5299 S, *I, *PartId, *TaskT, EKind, CodeGen, Data.Tied, Data.NumberOfParts);
5300 OMPLexicalScope Scope(*this, S, std::nullopt,
5303 TaskGen(*this, OutlinedFn, Data);
5304}
5305
5314 false, Loc, Ty, VK_LValue);
5319 false, Loc, Ty, VK_LValue);
5320 QualType ElemType = C.getBaseElementType(Ty);
5325 false, Loc, ElemType, VK_LValue);
5328 InitRef, nullptr,
5330 Data.FirstprivateVars.emplace_back(OrigRef);
5331 Data.FirstprivateCopies.emplace_back(PrivateRef);
5332 Data.FirstprivateInits.emplace_back(InitRef);
5333 return OrigVD;
5334}
5335
5339
5340 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
5345 auto PartId = std::next(I);
5346 auto TaskT = std::next(I, 4);
5348
5349 Data.Final.setInt(false);
5350
5352 auto IRef = C->varlist_begin();
5353 auto IElemInitRef = C->inits().begin();
5354 for (auto *IInit : C->private_copies()) {
5355 Data.FirstprivateVars.push_back(*IRef);
5356 Data.FirstprivateCopies.push_back(IInit);
5357 Data.FirstprivateInits.push_back(*IElemInitRef);
5358 ++IRef;
5359 ++IElemInitRef;
5360 }
5361 }
5365 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
5366 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5367 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
5368 Data.ReductionOps.append(C->reduction_ops().begin(),
5369 C->reduction_ops().end());
5370 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5371 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5372 }
5374 VarDecl *BPVD = nullptr;
5384 0);
5386 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5388 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5390 getContext().getIntTypeForBitwidth(64, 1),
5392 0);
5394 S.getBeginLoc());
5398
5399
5400 if (!isa_and_nonnullllvm::ConstantPointerNull(
5403 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5405 }
5406 }
5410 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD, EKind,
5411 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
5412
5414 if (.FirstprivateVars.empty()) {
5415 enum { PrivatesParam = 2, CopyFnParam = 3 };
5416 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5418 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5420
5424 CallArgs.push_back(PrivatesPtr);
5425 ParamTypes.push_back(PrivatesPtr->getType());
5426 for (const Expr *E : Data.FirstprivateVars) {
5429 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
5430 ".firstpriv.ptr.addr");
5431 PrivatePtrs.emplace_back(VD, PrivatePtr);
5432 CallArgs.push_back(PrivatePtr.getPointer());
5433 ParamTypes.push_back(PrivatePtr.getType());
5434 }
5435 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5436 ParamTypes, false);
5437 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5438 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5439 for (const auto &Pair : PrivatePtrs) {
5441 CGF.Builder.CreateLoad(Pair.second),
5442 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5443 CGF.getContext().getDeclAlign(Pair.first));
5444 Scope.addPrivate(Pair.first, Replacement);
5445 }
5446 }
5447 CGF.processInReduction(S, Data, CGF, CS, Scope);
5450 CGF.GetAddrOfLocalVar(BPVD), 0);
5451 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
5452 CGF.GetAddrOfLocalVar(PVD), 0);
5453 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
5454 CGF.GetAddrOfLocalVar(SVD), 0);
5455
5456 if (MVD)
5457 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(
5458 CGF.GetAddrOfLocalVar(MVD), 0);
5459 }
5460
5461 Action.Enter(CGF);
5462 OMPLexicalScope LexScope(CGF, S, OMPD_task, false);
5464 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
5466
5467
5468
5469 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
5470 CGF, TL->getThreadLimit().front(), S.getBeginLoc());
5471 }
5472 BodyGen(CGF);
5473 };
5474 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5475 S, *I, *PartId, *TaskT, EKind, CodeGen, true,
5476 Data.NumberOfParts);
5477 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
5479 getContext().getIntTypeForBitwidth(32, 0),
5481 CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
5482 SharedsTy, CapturedStruct, &IfCond, Data);
5483}
5484
5487 CodeGenFunction &CGF,
5491 if (Data.Reductions) {
5493 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5495 Data.ReductionCopies, Data.ReductionOps);
5498 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5501
5502
5503
5505 RedCG, Cnt);
5507 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5512 Data.ReductionCopies[Cnt]->getType()),
5513 Data.ReductionCopies[Cnt]->getExprLoc()),
5515 Replacement.getAlignment());
5518 }
5519 }
5520 (void)Scope.Privatize();
5526 auto IPriv = C->privates().begin();
5527 auto IRed = C->reduction_ops().begin();
5528 auto ITD = C->taskgroup_descriptors().begin();
5529 for (const Expr *Ref : C->varlist()) {
5530 InRedVars.emplace_back(Ref);
5531 InRedPrivs.emplace_back(*IPriv);
5532 InRedOps.emplace_back(*IRed);
5533 TaskgroupDescriptors.emplace_back(*ITD);
5534 std::advance(IPriv, 1);
5535 std::advance(IRed, 1);
5536 std::advance(ITD, 1);
5537 }
5538 }
5540 if (!InRedVars.empty()) {
5541 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
5542 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5545
5546
5547
5549 RedCG, Cnt);
5550 llvm::Value *ReductionsPtr;
5551 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5552 ReductionsPtr =
5554 } else {
5555 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5556 }
5558 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5563 InRedPrivs[Cnt]->getExprLoc()),
5565 Replacement.getAlignment());
5568 }
5569 }
5571}
5572
5574
5579 const Expr *IfCond = nullptr;
5580 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
5581 if (C->getNameModifier() == OMPD_unknown ||
5582 C->getNameModifier() == OMPD_task) {
5583 IfCond = C->getCondition();
5584 break;
5585 }
5586 }
5587
5589
5591 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
5593 };
5594 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5595 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
5598 SharedsTy, CapturedStruct, IfCond,
5600 };
5601 auto LPCRegion =
5604}
5605
5607 const OMPTaskyieldDirective &S) {
5608 CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
5609}
5610
5615 bool IsFatal = false;
5616 if (!SC || SC->getSeverityKind() == OMPC_SEVERITY_fatal)
5617 IsFatal = true;
5618 CGM.getOpenMPRuntime().emitErrorCall(*this, S.getBeginLoc(), ME, IsFatal);
5619}
5620
5622 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
5623}
5624
5627
5630 CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(), Data);
5631}
5632
5634 return T.clauses().empty();
5635}
5636
5638 const OMPTaskgroupDirective &S) {
5639 OMPLexicalScope Scope(*this, S, OMPD_unknown);
5641 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5642 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5645
5646 auto BodyGenCB = [&, this](InsertPointTy AllocaIP,
5647 InsertPointTy CodeGenIP) {
5648 Builder.restoreIP(CodeGenIP);
5649 EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5650 return llvm::Error::success();
5651 };
5655 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5656 cantFail(OMPBuilder.createTaskgroup(Builder, AllocaIP, BodyGenCB));
5657 Builder.restoreIP(AfterIP);
5658 return;
5659 }
5661 Action.Enter(CGF);
5662 if (const Expr *E = S.getReductionRef()) {
5667 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
5668 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5669 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
5670 Data.ReductionOps.append(C->reduction_ops().begin(),
5671 C->reduction_ops().end());
5672 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5673 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5674 }
5675 llvm::Value *ReductionDesc =
5677 LHSs, RHSs, Data);
5681 false, E->getType());
5682 }
5683 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5684 };
5685 CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
5686}
5687
5689 llvm::AtomicOrdering AO = S.getSingleClause<OMPFlushClause>()
5690 ? llvm::AtomicOrdering::NotAtomic
5691 : llvm::AtomicOrdering::AcquireRelease;
5692 CGM.getOpenMPRuntime().emitFlush(
5693 *this,
5695 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
5697 FlushClause->varlist_end());
5698 return {};
5699 }(),
5700 S.getBeginLoc(), AO);
5701}
5702
5706 if (const auto *DC = S.getSingleClause<OMPDependClause>()) {
5707
5710 for (auto &Dep : Data.Dependences) {
5711 Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause(
5712 *this, Dep, DC->getBeginLoc());
5714 }
5715 return;
5716 }
5717 if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) {
5718 CGM.getOpenMPRuntime().emitDestroyClause(*this, DOLVal, DC->getBeginLoc());
5719 return;
5720 }
5721 if (const auto *UC = S.getSingleClause<OMPUpdateClause>()) {
5722 CGM.getOpenMPRuntime().emitUpdateClause(
5723 *this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());
5724 return;
5725 }
5726}
5727
5730 return;
5741 for (const auto *C : ParentDir.getClausesOfKind<OMPReductionClause>()) {
5742 if (C->getModifier() != OMPC_REDUCTION_inscan)
5743 continue;
5744 Shareds.append(C->varlist_begin(), C->varlist_end());
5745 Privates.append(C->privates().begin(), C->privates().end());
5746 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5747 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5748 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
5749 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
5750 CopyArrayTemps.append(C->copy_array_temps().begin(),
5751 C->copy_array_temps().end());
5752 CopyArrayElems.append(C->copy_array_elems().begin(),
5753 C->copy_array_elems().end());
5754 }
5755 if (ParentDir.getDirectiveKind() == OMPD_simd ||
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794 llvm::BasicBlock *OMPScanReduce = createBasicBlock("omp.inscan.reduce");
5796 ? OMPScanReduce
5797 : BreakContinueStack.back().ContinueBlock.getBlock());
5799 {
5800
5801
5805 if (!IsInclusive) {
5806
5807
5808 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5809 const Expr *PrivateExpr = Privates[I];
5810 const Expr *TempExpr = CopyArrayTemps[I];
5819 CopyOps[I]);
5820 }
5821 }
5822 CGM.getOpenMPRuntime().emitReduction(
5823 *this, ParentDir.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
5824 {true, true,
5825 {}, OMPD_simd});
5826 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5827 const Expr *PrivateExpr = Privates[I];
5830 if (IsInclusive) {
5833 } else {
5834 const Expr *TempExpr = CopyArrayTemps[I];
5837 }
5842 }
5843 }
5846 ? BreakContinueStack.back().ContinueBlock.getBlock()
5847 : OMPScanReduce;
5849 return;
5850 }
5851 if (!IsInclusive) {
5852 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5854 }
5856
5858 .getIterationVariable()
5859 ->IgnoreParenImpCasts();
5862 IdxVal = Builder.CreateIntCast(IdxVal, SizeTy, false);
5863 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5864 const Expr *PrivateExpr = Privates[I];
5865 const Expr *OrigExpr = Shareds[I];
5866 const Expr *CopyArrayElem = CopyArrayElems[I];
5867 OpaqueValueMapping IdxMapping(
5868 *this,
5878 }
5879 }
5880 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5881 if (IsInclusive) {
5883 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5884 }
5887
5889 .getIterationVariable()
5890 ->IgnoreParenImpCasts();
5891 LValue IdxLVal = EmitLValue(IVExpr);
5893 IdxVal = Builder.CreateIntCast(IdxVal, SizeTy, false);
5894 llvm::BasicBlock *ExclusiveExitBB = nullptr;
5895 if (!IsInclusive) {
5896 llvm::BasicBlock *ContBB = createBasicBlock("omp.exclusive.dec");
5897 ExclusiveExitBB = createBasicBlock("omp.exclusive.copy.exit");
5898 llvm::Value *Cmp = Builder.CreateIsNull(IdxVal);
5899 Builder.CreateCondBr(Cmp, ExclusiveExitBB, ContBB);
5901
5902 IdxVal = Builder.CreateNUWSub(IdxVal, llvm::ConstantInt::get(SizeTy, 1));
5903 }
5904 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5905 const Expr *PrivateExpr = Privates[I];
5906 const Expr *OrigExpr = Shareds[I];
5907 const Expr *CopyArrayElem = CopyArrayElems[I];
5909 *this,
5913 LValue SrcLVal = EmitLValue(CopyArrayElem);
5914 LValue DestLVal = EmitLValue(OrigExpr);
5916 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
5919 }
5920 if (!IsInclusive) {
5922 }
5923 }
5927}
5928
5931 Expr *IncExpr) {
5932
5933 const auto *IVExpr = cast(S.getIterationVariable());
5934 const auto *IVDecl = cast(IVExpr->getDecl());
5936
5937
5938
5939
5940 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) {
5942
5944 }
5945
5947
5948 bool HasLastprivateClause = false;
5949
5950 {
5951 OMPLoopScope PreInitScope(*this, S);
5952
5953
5954
5955 bool CondConstant;
5956 llvm::BasicBlock *ContBlock = nullptr;
5958 if (!CondConstant)
5959 return;
5960 } else {
5961 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
5963 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
5967 }
5968
5970
5971 {
5972
5973
5977 ? S.getCombinedLowerBoundVariable()
5978 : S.getLowerBoundVariable())));
5982 ? S.getCombinedUpperBoundVariable()
5983 : S.getUpperBoundVariable())));
5988
5991
5992
5993
5994 CGM.getOpenMPRuntime().emitBarrierCall(
5995 *this, S.getBeginLoc(), OMPD_unknown, false,
5996 true);
5997 }
6007 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
6008
6009
6010 llvm::Value *Chunk = nullptr;
6013 ScheduleKind = C->getDistScheduleKind();
6014 if (const Expr *Ch = C->getChunkSize()) {
6017 S.getIterationVariable()->getType(),
6018 S.getBeginLoc());
6019 }
6020 } else {
6021
6022 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
6023 *this, S, ScheduleKind, Chunk);
6024 }
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036 bool StaticChunked =
6037 RT.isStaticChunked(ScheduleKind, Chunk != nullptr) &&
6040 Chunk != nullptr) ||
6041 StaticChunked) {
6043 IVSize, IVSigned, false, IL.getAddress(),
6045 StaticChunked ? Chunk : nullptr);
6047 StaticInit);
6050
6052 ? S.getCombinedEnsureUpperBound()
6053 : S.getEnsureUpperBound());
6054
6056 ? S.getCombinedInit()
6057 : S.getInit());
6058
6061 ? S.getCombinedCond()
6062 : S.getCond();
6063
6064 if (StaticChunked)
6065 Cond = S.getCombinedDistCond();
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6092 *this, S,
6096 },
6097 [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,
6098 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
6100 S, LoopScope.requiresCleanups(), Cond, IncExpr,
6101 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
6102 CodeGenLoop(CGF, S, LoopExit);
6103 },
6104 [&S, StaticChunked](CodeGenFunction &CGF) {
6105 if (StaticChunked) {
6106 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
6107 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
6108 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
6109 CGF.EmitIgnoredExpr(S.getCombinedInit());
6110 }
6111 });
6112 });
6114
6116 } else {
6117
6118
6119 const OMPLoopArguments LoopArguments = {
6120 LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
6121 Chunk};
6122 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
6123 CodeGenLoop);
6124 }
6127 return CGF.Builder.CreateIsNotNull(
6129 });
6130 }
6135
6137 *this, S, [IL, &S](CodeGenFunction &CGF) {
6138 return CGF.Builder.CreateIsNotNull(
6140 });
6141 }
6142
6143 if (HasLastprivateClause) {
6145 S, false,
6147 }
6148 }
6149
6150
6151 if (ContBlock) {
6154 }
6155 }
6156}
6157
6158
6159
6165 };
6166 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
6168}
6169
6174
6175static llvm::Function *
6182 Fn->setDoesNotRecurse();
6183 return Fn;
6184}
6185
6186template
6188 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
6189 llvm::OpenMPIRBuilder &OMPBuilder) {
6190
6191 unsigned NumLoops = C->getNumLoops();
6193 64, 1);
6195 for (unsigned I = 0; I < NumLoops; I++) {
6196 const Expr *CounterVal = C->getLoopData(I);
6197 assert(CounterVal);
6201 StoreValues.emplace_back(StoreValue);
6202 }
6203 OMPDoacrossKind ODK;
6204 bool IsDependSource = ODK.isSource(C);
6206 OMPBuilder.createOrderedDepend(CGF.Builder, AllocaIP, NumLoops,
6207 StoreValues, ".cnt.addr", IsDependSource));
6208}
6209
6211 if (CGM.getLangOpts().OpenMPIRBuilder) {
6212 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6213 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6214
6217
6218 assert(!S.hasAssociatedStmt() && "No associated statement must be in "
6219 "ordered depend|doacross construct.");
6222 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
6226 } else {
6227
6228
6230
6231 auto FiniCB = [this](InsertPointTy IP) {
6233 return llvm::Error::success();
6234 };
6235
6236 auto BodyGenCB = [&S, C, this](InsertPointTy AllocaIP,
6237 InsertPointTy CodeGenIP) {
6238 Builder.restoreIP(CodeGenIP);
6239
6240 const CapturedStmt *CS = S.getInnermostCapturedStmt();
6241 if (C) {
6242 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
6243 Builder, false, ".ordered.after");
6247 assert(S.getBeginLoc().isValid() &&
6248 "Outlined function call location must be valid.");
6251 OutlinedFn, CapturedVars);
6252 } else {
6254 *this, CS->getCapturedStmt(), AllocaIP, CodeGenIP, "ordered");
6255 }
6256 return llvm::Error::success();
6257 };
6258
6259 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6260 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
6261 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, ));
6262 Builder.restoreIP(AfterIP);
6263 }
6264 return;
6265 }
6266
6268 assert(!S.hasAssociatedStmt() &&
6269 "No associated statement must be in ordered depend construct.");
6270 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
6271 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
6272 return;
6273 }
6275 assert(!S.hasAssociatedStmt() &&
6276 "No associated statement must be in ordered doacross construct.");
6278 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
6279 return;
6280 }
6282 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
6284 const CapturedStmt *CS = S.getInnermostCapturedStmt();
6285 if (C) {
6287 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
6289 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
6290 OutlinedFn, CapturedVars);
6291 } else {
6292 Action.Enter(CGF);
6294 }
6295 };
6296 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6297 CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), );
6298}
6299
6304 "DestType must have scalar evaluation kind.");
6305 assert(!Val.isAggregate() && "Must be a scalar or complex.");
6307 DestType, Loc)
6310}
6311
6316 "DestType must have complex evaluation kind.");
6319
6323 Val.getScalarVal(), SrcType, DestElementType, Loc);
6325 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
6326 } else {
6327 assert(Val.isComplex() && "Must be a scalar or complex.");
6332 Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
6334 Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
6335 }
6336 return ComplexVal;
6337}
6338
6340 LValue LVal, RValue RVal) {
6341 if (LVal.isGlobalReg())
6343 else
6344 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), false);
6345}
6346
6348 llvm::AtomicOrdering AO, LValue LVal,
6350 if (LVal.isGlobalReg())
6353 LVal, Loc, llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO),
6354 LVal.isVolatile());
6355}
6356
6362 *this, RVal, RValTy, LVal.getType(), Loc)),
6363 LVal);
6364 break;
6368 false);
6369 break;
6371 llvm_unreachable("Must be a scalar or complex.");
6372 }
6373}
6374
6378
6379 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
6380 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
6384
6385
6386
6387
6388 switch (AO) {
6389 case llvm::AtomicOrdering::Acquire:
6390 case llvm::AtomicOrdering::AcquireRelease:
6391 case llvm::AtomicOrdering::SequentiallyConsistent:
6393 llvm::AtomicOrdering::Acquire);
6394 break;
6395 case llvm::AtomicOrdering::Monotonic:
6396 case llvm::AtomicOrdering::Release:
6397 break;
6398 case llvm::AtomicOrdering::NotAtomic:
6399 case llvm::AtomicOrdering::Unordered:
6400 llvm_unreachable("Unexpected ordering.");
6401 }
6402 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
6404}
6405
6407 llvm::AtomicOrdering AO, const Expr *X,
6409
6410 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
6413
6414
6415
6416
6417 switch (AO) {
6418 case llvm::AtomicOrdering::Release:
6419 case llvm::AtomicOrdering::AcquireRelease:
6420 case llvm::AtomicOrdering::SequentiallyConsistent:
6422 llvm::AtomicOrdering::Release);
6423 break;
6424 case llvm::AtomicOrdering::Acquire:
6425 case llvm::AtomicOrdering::Monotonic:
6426 break;
6427 case llvm::AtomicOrdering::NotAtomic:
6428 case llvm::AtomicOrdering::Unordered:
6429 llvm_unreachable("Unexpected ordering.");
6430 }
6431}
6432
6436 llvm::AtomicOrdering AO,
6439
6440
6441
6442 if (BO == BO_Comma || .isScalar() ||
.isSimple() ||
6444 (Update.getScalarVal()->getType() != X.getAddress().getElementType())) ||
6445 !Context.getTargetInfo().hasBuiltinAtomic(
6446 Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
6447 return std::make_pair(false, RValue::get(nullptr));
6448
6449 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) {
6450 if (T->isIntegerTy())
6451 return true;
6452
6453 if (T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
6454 return llvm::isPowerOf2_64(CGF.CGM.getDataLayout().getTypeStoreSize(T));
6455
6456 return false;
6457 };
6458
6459 if (!CheckAtomicSupport(Update.getScalarVal()->getType(), BO) ||
6460 !CheckAtomicSupport(X.getAddress().getElementType(), BO))
6461 return std::make_pair(false, RValue::get(nullptr));
6462
6463 bool IsInteger = X.getAddress().getElementType()->isIntegerTy();
6464 llvm::AtomicRMWInst::BinOp RMWOp;
6465 switch (BO) {
6466 case BO_Add:
6467 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
6468 break;
6469 case BO_Sub:
6471 return std::make_pair(false, RValue::get(nullptr));
6472 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
6473 break;
6474 case BO_And:
6475 RMWOp = llvm::AtomicRMWInst::And;
6476 break;
6477 case BO_Or:
6478 RMWOp = llvm::AtomicRMWInst::Or;
6479 break;
6480 case BO_Xor:
6481 RMWOp = llvm::AtomicRMWInst::Xor;
6482 break;
6483 case BO_LT:
6484 if (IsInteger)
6485 RMWOp = X.getType()->hasSignedIntegerRepresentation()
6487 : llvm::AtomicRMWInst::Max)
6489 : llvm::AtomicRMWInst::UMax);
6490 else
6492 : llvm::AtomicRMWInst::FMax;
6493 break;
6494 case BO_GT:
6495 if (IsInteger)
6496 RMWOp = X.getType()->hasSignedIntegerRepresentation()
6498 : llvm::AtomicRMWInst::Min)
6500 : llvm::AtomicRMWInst::UMin);
6501 else
6503 : llvm::AtomicRMWInst::FMin;
6504 break;
6505 case BO_Assign:
6506 RMWOp = llvm::AtomicRMWInst::Xchg;
6507 break;
6508 case BO_Mul:
6509 case BO_Div:
6510 case BO_Rem:
6511 case BO_Shl:
6512 case BO_Shr:
6513 case BO_LAnd:
6514 case BO_LOr:
6515 return std::make_pair(false, RValue::get(nullptr));
6516 case BO_PtrMemD:
6517 case BO_PtrMemI:
6518 case BO_LE:
6519 case BO_GE:
6520 case BO_EQ:
6521 case BO_NE:
6522 case BO_Cmp:
6523 case BO_AddAssign:
6524 case BO_SubAssign:
6525 case BO_AndAssign:
6526 case BO_OrAssign:
6527 case BO_XorAssign:
6528 case BO_MulAssign:
6529 case BO_DivAssign:
6530 case BO_RemAssign:
6531 case BO_ShlAssign:
6532 case BO_ShrAssign:
6533 case BO_Comma:
6534 llvm_unreachable("Unsupported atomic update operation");
6535 }
6536 llvm::Value *UpdateVal = Update.getScalarVal();
6537 if (auto *IC = dyn_castllvm::ConstantInt(UpdateVal)) {
6538 if (IsInteger)
6539 UpdateVal = CGF.Builder.CreateIntCast(
6540 IC, X.getAddress().getElementType(),
6541 X.getType()->hasSignedIntegerRepresentation());
6542 else
6543 UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
6544 X.getAddress().getElementType());
6545 }
6546 llvm::AtomicRMWInst *Res =
6548 return std::make_pair(true, RValue::get(Res));
6549}
6550
6554 const llvm::function_ref<RValue(RValue)> CommonGen) {
6555
6556
6557
6558
6559
6560
6562 if (!Res.first) {
6563 if (X.isGlobalReg()) {
6564
6565
6567 } else {
6568
6569 EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
6570 }
6571 }
6572 return Res;
6573}
6574
6576 llvm::AtomicOrdering AO, const Expr *X,
6577 const Expr *E, const Expr *UE,
6580 "Update expr in 'atomic update' must be a binary operator.");
6582
6583
6584
6585
6586
6587
6588 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
6595 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
6599 };
6601 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
6603
6604
6605
6606
6607 switch (AO) {
6608 case llvm::AtomicOrdering::Release:
6609 case llvm::AtomicOrdering::AcquireRelease:
6610 case llvm::AtomicOrdering::SequentiallyConsistent:
6612 llvm::AtomicOrdering::Release);
6613 break;
6614 case llvm::AtomicOrdering::Acquire:
6615 case llvm::AtomicOrdering::Monotonic:
6616 break;
6617 case llvm::AtomicOrdering::NotAtomic:
6618 case llvm::AtomicOrdering::Unordered:
6619 llvm_unreachable("Unexpected ordering.");
6620 }
6621}
6622
6633 }
6635 break;
6636 }
6637 llvm_unreachable("Must be a scalar or complex.");
6638}
6639
6641 llvm::AtomicOrdering AO,
6646 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
6647 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
6653 if (UE) {
6654
6656 "Update expr in 'atomic capture' must be a binary operator.");
6658
6659
6660
6661
6662
6663
6667 NewVValType = XRValExpr->getType();
6669 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
6675 return Res;
6676 };
6678 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
6680 if (Res.first) {
6681
6683
6684 NewVVal = Res.second;
6685 } else {
6686
6687
6691 }
6692 }
6693 } else {
6694
6695 NewVValType = X->getType().getNonReferenceType();
6697 X->getType().getNonReferenceType(), Loc);
6698 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
6699 NewVVal = XRValue;
6700 return ExprRValue;
6701 };
6702
6704 XLValue, ExprRValue, BO_Assign, false, AO,
6705 Loc, Gen);
6707 if (Res.first) {
6708
6710 }
6711 }
6712
6715
6717
6718
6719
6720
6721
6722
6723
6724 switch (AO) {
6725 case llvm::AtomicOrdering::Release:
6727 llvm::AtomicOrdering::Release);
6728 break;
6729 case llvm::AtomicOrdering::Acquire:
6731 llvm::AtomicOrdering::Acquire);
6732 break;
6733 case llvm::AtomicOrdering::AcquireRelease:
6734 case llvm::AtomicOrdering::SequentiallyConsistent:
6736 CGF, {}, Loc, llvm::AtomicOrdering::AcquireRelease);
6737 break;
6738 case llvm::AtomicOrdering::Monotonic:
6739 break;
6740 case llvm::AtomicOrdering::NotAtomic:
6741 case llvm::AtomicOrdering::Unordered:
6742 llvm_unreachable("Unexpected ordering.");
6743 }
6744 }
6745}
6746
6748 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
6752 llvm::OpenMPIRBuilder &OMPBuilder =
6754
6755 OMPAtomicCompareOp Op;
6758 case BO_EQ:
6759 Op = OMPAtomicCompareOp::EQ;
6760 break;
6761 case BO_LT:
6762 Op = OMPAtomicCompareOp::MIN;
6763 break;
6764 case BO_GT:
6765 Op = OMPAtomicCompareOp::MAX;
6766 break;
6767 default:
6768 llvm_unreachable("unsupported atomic compare binary operator");
6769 }
6770
6772 Address XAddr = XLVal.getAddress();
6773
6774 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) {
6775 if (X->getType() == E->getType())
6779 if (NewE->getType() == X->getType())
6780 return V;
6782 };
6783
6784 llvm::Value *EVal = EmitRValueWithCastIfNeeded(X, E);
6785 llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(X, D) : nullptr;
6786 if (auto *CI = dyn_castllvm::ConstantInt(EVal))
6787 EVal = CGF.Builder.CreateIntCast(
6788 CI, XLVal.getAddress().getElementType(),
6790 if (DVal)
6791 if (auto *CI = dyn_castllvm::ConstantInt(DVal))
6792 DVal = CGF.Builder.CreateIntCast(
6793 CI, XLVal.getAddress().getElementType(),
6795
6796 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
6798 X->getType()->hasSignedIntegerRepresentation(),
6799 X->getType().isVolatileQualified()};
6800 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
6801 if (V) {
6804 VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
6805 V->getType()->hasSignedIntegerRepresentation(),
6806 V->getType().isVolatileQualified()};
6807 }
6808 if (R) {
6811 ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
6814 }
6815
6816 if (FailAO == llvm::AtomicOrdering::NotAtomic) {
6817
6818
6819 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
6820 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6822 } else
6823 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
6824 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6826}
6827
6829 llvm::AtomicOrdering AO,
6832 const Expr *E, const Expr *UE, const Expr *D,
6835 switch (Kind) {
6836 case OMPC_read:
6838 break;
6839 case OMPC_write:
6841 break;
6842 case OMPC_unknown:
6843 case OMPC_update:
6845 break;
6846 case OMPC_capture:
6849 break;
6850 case OMPC_compare: {
6853 break;
6854 }
6855 default:
6856 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
6857 }
6858}
6859
6861 llvm::AtomicOrdering AO = CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
6862
6863 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
6864 bool MemOrderingSpecified = false;
6866 AO = llvm::AtomicOrdering::SequentiallyConsistent;
6867 MemOrderingSpecified = true;
6869 AO = llvm::AtomicOrdering::AcquireRelease;
6870 MemOrderingSpecified = true;
6872 AO = llvm::AtomicOrdering::Acquire;
6873 MemOrderingSpecified = true;
6875 AO = llvm::AtomicOrdering::Release;
6876 MemOrderingSpecified = true;
6878 AO = llvm::AtomicOrdering::Monotonic;
6879 MemOrderingSpecified = true;
6880 }
6881 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
6883 for (const OMPClause *C : S.clauses()) {
6884
6885
6887
6888 if (K == OMPC_weak)
6889 return;
6890 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
6891 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
6892 continue;
6893 Kind = K;
6894 KindsEncountered.insert(K);
6895 }
6896
6897
6898
6899 if (KindsEncountered.contains(OMPC_compare) &&
6900 KindsEncountered.contains(OMPC_capture))
6901 Kind = OMPC_compare;
6902 if (!MemOrderingSpecified) {
6903 llvm::AtomicOrdering DefaultOrder =
6904 CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
6905 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
6906 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
6907 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
6908 Kind == OMPC_capture)) {
6909 AO = DefaultOrder;
6910 } else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
6911 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
6912 AO = llvm::AtomicOrdering::Release;
6913 } else if (Kind == OMPC_read) {
6914 assert(Kind == OMPC_read && "Unexpected atomic kind.");
6915 AO = llvm::AtomicOrdering::Acquire;
6916 }
6917 }
6918 }
6919
6920 if (KindsEncountered.contains(OMPC_compare) &&
6921 KindsEncountered.contains(OMPC_fail)) {
6922 Kind = OMPC_compare;
6923 const auto *FailClause = S.getSingleClause<OMPFailClause>();
6924 if (FailClause) {
6925 OpenMPClauseKind FailParameter = FailClause->getFailParameter();
6926 if (FailParameter == llvm::omp::OMPC_relaxed)
6927 FailAO = llvm::AtomicOrdering::Monotonic;
6928 else if (FailParameter == llvm::omp::OMPC_acquire)
6929 FailAO = llvm::AtomicOrdering::Acquire;
6930 else if (FailParameter == llvm::omp::OMPC_seq_cst)
6931 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
6932 }
6933 }
6934
6937 emitOMPAtomicExpr(*this, Kind, AO, FailAO, S.isPostfixUpdate(), S.getX(),
6938 S.getV(), S.getR(), S.getExpr(), S.getUpdateExpr(),
6939 S.getD(), S.getCondExpr(), S.isXLHSInRHSPart(),
6940 S.isFailOnly(), S.getBeginLoc());
6941}
6942
6948
6949
6951 OMPLexicalScope Scope(CGF, S, OMPD_target);
6954 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6955 });
6956 return;
6957 }
6958
6960 llvm::Function *Fn = nullptr;
6961 llvm::Constant *FnID = nullptr;
6962
6963 const Expr *IfCond = nullptr;
6964
6965 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
6966 if (C->getNameModifier() == OMPD_unknown ||
6967 C->getNameModifier() == OMPD_target) {
6968 IfCond = C->getCondition();
6969 break;
6970 }
6971 }
6972
6973
6974 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device(
6977 Device.setPointerAndInt(C->getDevice(), C->getModifier());
6978
6979
6980
6981
6982 bool IsOffloadEntry = true;
6983 if (IfCond) {
6984 bool Val;
6986 IsOffloadEntry = false;
6987 }
6989 IsOffloadEntry = false;
6990
6991 if (CGM.getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
6994 "No offloading entry generated while offloading is mandatory.");
6996 }
6997
6998 assert(CGF.CurFuncDecl && "No parent declaration for target region!");
6999 StringRef ParentName;
7000
7001
7002 if (const auto *D = dyn_cast(CGF.CurFuncDecl))
7004 else if (const auto *D = dyn_cast(CGF.CurFuncDecl))
7006 else
7007 ParentName =
7009
7010
7012 IsOffloadEntry, CodeGen);
7013 OMPLexicalScope Scope(CGF, S, OMPD_task);
7014 auto &&SizeEmitter =
7017 if (IsOffloadEntry) {
7018 OMPLoopScope(CGF, D);
7019
7020 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
7021 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
7022 false);
7023 return NumIterations;
7024 }
7025 return nullptr;
7026 };
7028 SizeEmitter);
7029}
7030
7033 Action.Enter(CGF);
7040
7041 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
7043}
7044
7046 StringRef ParentName,
7050 };
7051 llvm::Function *Fn;
7052 llvm::Constant *Addr;
7053
7054 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7055 S, ParentName, Fn, Addr, true, CodeGen);
7056 assert(Fn && Addr && "Target device function emission failed.");
7057}
7058
7065
7070 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
7071 llvm::Function *OutlinedFn =
7075
7078 if (NT || TL) {
7079 const Expr *NumTeams = NT ? NT->getNumTeams().front() : nullptr;
7080 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() : nullptr;
7081
7083 S.getBeginLoc());
7084 }
7085
7086 OMPTeamsScope Scope(CGF, S);
7090 CapturedVars);
7091}
7092
7094
7096 Action.Enter(CGF);
7102 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
7104 };
7107 [](CodeGenFunction &) { return nullptr; });
7108}
7109
7112 auto *CS = S.getCapturedStmt(OMPD_teams);
7113 Action.Enter(CGF);
7114
7116 Action.Enter(CGF);
7126 };
7130}
7131
7137 };
7138 llvm::Function *Fn;
7139 llvm::Constant *Addr;
7140
7141 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7142 S, ParentName, Fn, Addr, true, CodeGen);
7143 assert(Fn && Addr && "Target device function emission failed.");
7144}
7145
7153
7154static void
7157 Action.Enter(CGF);
7160 };
7161
7162
7165 Action.Enter(CGF);
7170 CodeGenDistribute);
7172 };
7176}
7177
7183 };
7184 llvm::Function *Fn;
7185 llvm::Constant *Addr;
7186
7187 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7188 S, ParentName, Fn, Addr, true, CodeGen);
7189 assert(Fn && Addr && "Target device function emission failed.");
7190}
7191
7199
7203 Action.Enter(CGF);
7206 };
7207
7208
7211 Action.Enter(CGF);
7216 CodeGenDistribute);
7218 };
7222}
7223
7229 };
7230 llvm::Function *Fn;
7231 llvm::Constant *Addr;
7232
7233 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7234 S, ParentName, Fn, Addr, true, CodeGen);
7235 assert(Fn && Addr && "Target device function emission failed.");
7236}
7237
7245
7248
7249 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7251 };
7252
7253
7254 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7256 Action.Enter(CGF);
7261 CodeGenDistribute);
7263 };
7266 [](CodeGenFunction &) { return nullptr; });
7267}
7268
7271 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7273 };
7274
7275
7276 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7278 Action.Enter(CGF);
7283 CodeGenDistribute);
7285 };
7288 [](CodeGenFunction &) { return nullptr; });
7289}
7290
7293 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7295 S.getDistInc());
7296 };
7297
7298
7299 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7301 Action.Enter(CGF);
7306 CodeGenDistribute);
7308 };
7311 [](CodeGenFunction &) { return nullptr; });
7312}
7313
7316 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7318 S.getDistInc());
7319 };
7320
7321
7322 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7324 Action.Enter(CGF);
7329 CGF, OMPD_distribute, CodeGenDistribute, false);
7331 };
7335 [](CodeGenFunction &) { return nullptr; });
7336}
7337
7339 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
7340 llvm::Value *Device = nullptr;
7341 llvm::Value *NumDependences = nullptr;
7342 llvm::Value *DependenceList = nullptr;
7343
7346
7347
7350 if (.Dependences.empty()) {
7352 std::tie(NumDependences, DependenciesArray) =
7353 CGM.getOpenMPRuntime().emitDependClause(*this, Data.Dependences,
7354 S.getBeginLoc());
7355 DependenceList = DependenciesArray.emitRawPointer(*this);
7356 }
7358
7359 assert(!(Data.HasNowaitClause && !(S.getSingleClause<OMPInitClause>() ||
7362 "OMPNowaitClause clause is used separately in OMPInteropDirective.");
7363
7364 auto ItOMPInitClause = S.getClausesOfKind<OMPInitClause>();
7365 if (!ItOMPInitClause.empty()) {
7366
7368 llvm::Value *InteropvarPtr =
7370 llvm::omp::OMPInteropType InteropType =
7371 llvm::omp::OMPInteropType::Unknown;
7372 if (C->getIsTarget()) {
7373 InteropType = llvm::omp::OMPInteropType::Target;
7374 } else {
7375 assert(C->getIsTargetSync() &&
7376 "Expected interop-type target/targetsync");
7377 InteropType = llvm::omp::OMPInteropType::TargetSync;
7378 }
7379 OMPBuilder.createOMPInteropInit(Builder, InteropvarPtr, InteropType,
7380 Device, NumDependences, DependenceList,
7381 Data.HasNowaitClause);
7382 }
7383 }
7384 auto ItOMPDestroyClause = S.getClausesOfKind<OMPDestroyClause>();
7385 if (!ItOMPDestroyClause.empty()) {
7386
7388 llvm::Value *InteropvarPtr =
7390 OMPBuilder.createOMPInteropDestroy(Builder, InteropvarPtr, Device,
7391 NumDependences, DependenceList,
7392 Data.HasNowaitClause);
7393 }
7394 }
7395 auto ItOMPUseClause = S.getClausesOfKind<OMPUseClause>();
7396 if (!ItOMPUseClause.empty()) {
7397
7399 llvm::Value *InteropvarPtr =
7401 OMPBuilder.createOMPInteropUse(Builder, InteropvarPtr, Device,
7402 NumDependences, DependenceList,
7403 Data.HasNowaitClause);
7404 }
7405 }
7406}
7407
7411 Action.Enter(CGF);
7414 S.getDistInc());
7415 };
7416
7417
7418 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7420 Action.Enter(CGF);
7425 CGF, OMPD_distribute, CodeGenDistribute, false);
7427 };
7428
7430 CodeGenTeams);
7433}
7434
7438
7439
7442 };
7443 llvm::Function *Fn;
7444 llvm::Constant *Addr;
7445
7446 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7447 S, ParentName, Fn, Addr, true, CodeGen);
7448 assert(Fn && Addr && "Target device function emission failed.");
7449}
7450
7458
7463 Action.Enter(CGF);
7466 S.getDistInc());
7467 };
7468
7469
7470 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7472 Action.Enter(CGF);
7477 CGF, OMPD_distribute, CodeGenDistribute, false);
7479 };
7480
7482 CodeGenTeams);
7485}
7486
7490
7491
7494 };
7495 llvm::Function *Fn;
7496 llvm::Constant *Addr;
7497
7498 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7499 S, ParentName, Fn, Addr, true, CodeGen);
7500 assert(Fn && Addr && "Target device function emission failed.");
7501}
7502
7510
7513 CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
7515}
7516
7518 const Expr *IfCond = nullptr;
7519 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7520 if (C->getNameModifier() == OMPD_unknown ||
7521 C->getNameModifier() == OMPD_cancel) {
7522 IfCond = C->getCondition();
7523 break;
7524 }
7525 }
7526 if (CGM.getLangOpts().OpenMPIRBuilder) {
7527 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
7528
7529
7533 llvm::Value *IfCondition = nullptr;
7534 if (IfCond)
7536 true);
7537 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
7539 return Builder.restoreIP(AfterIP);
7540 }
7541 }
7542
7543 CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
7545}
7546
7549 if (Kind == OMPD_parallel || Kind == OMPD_task ||
7550 Kind == OMPD_target_parallel || Kind == OMPD_taskloop ||
7551 Kind == OMPD_master_taskloop || Kind == OMPD_parallel_master_taskloop)
7553 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
7554 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
7555 Kind == OMPD_distribute_parallel_for ||
7556 Kind == OMPD_target_parallel_for ||
7557 Kind == OMPD_teams_distribute_parallel_for ||
7558 Kind == OMPD_target_teams_distribute_parallel_for);
7559 return OMPCancelStack.getExitBlock();
7560}
7561
7564 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7565 CaptureDeviceAddrMap) {
7566 llvm::SmallDenseSet<CanonicalDeclPtr, 4> Processed;
7567 for (const Expr *OrigVarIt : C.varlist()) {
7569 if (!Processed.insert(OrigVD).second)
7570 continue;
7571
7572
7573
7574
7575 const ValueDecl *MatchingVD = OrigVD;
7576 if (const auto *OED = dyn_cast(MatchingVD)) {
7577
7578
7581 "Base should be the current struct!");
7582 MatchingVD = ME->getMemberDecl();
7583 }
7584
7585
7586
7587 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7588 if (InitAddrIt == CaptureDeviceAddrMap.end())
7589 continue;
7590
7591 llvm::Type *Ty = ConvertTypeForMem(OrigVD->getType().getNonReferenceType());
7592
7593
7594 bool IsRegistered = PrivateScope.addPrivate(
7595 OrigVD,
7596 Address(InitAddrIt->second, Ty,
7598 assert(IsRegistered && "firstprivate var already registered as private");
7599
7600 (void)IsRegistered;
7601 }
7602}
7603
7606 while (const auto *OASE = dyn_cast(Base))
7607 Base = OASE->getBase()->IgnoreParenImpCasts();
7608 while (const auto *ASE = dyn_cast(Base))
7609 Base = ASE->getBase()->IgnoreParenImpCasts();
7611}
7612
7615 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7616 CaptureDeviceAddrMap) {
7617 llvm::SmallDenseSet<CanonicalDeclPtr, 4> Processed;
7618 for (const Expr *Ref : C.varlist()) {
7620 if (!Processed.insert(OrigVD).second)
7621 continue;
7622
7623
7624
7625 const ValueDecl *MatchingVD = OrigVD;
7626 if (const auto *OED = dyn_cast(MatchingVD)) {
7627
7628
7631 "Base should be the current struct!");
7632 MatchingVD = ME->getMemberDecl();
7633 }
7634
7635
7636
7637 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7638 if (InitAddrIt == CaptureDeviceAddrMap.end())
7639 continue;
7640
7642
7644 Address(InitAddrIt->second, Ty,
7646
7647
7652 PrivAddr =
7655 }
7656
7657 (void)PrivateScope.addPrivate(OrigVD, PrivAddr);
7658 }
7659}
7660
7661
7665 true);
7666
7667
7668
7669
7670 bool PrivatizeDevicePointers = false;
7671 class DevicePointerPrivActionTy : public PrePostActionTy {
7672 bool &PrivatizeDevicePointers;
7673
7674 public:
7675 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
7676 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
7677 void Enter(CodeGenFunction &CGF) override {
7678 PrivatizeDevicePointers = true;
7679 }
7680 };
7681 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
7682
7684 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7685 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
7686 };
7687
7688
7689 auto &&PrivCodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
7691 PrivatizeDevicePointers = false;
7692
7693
7694
7695 Action.Enter(CGF);
7696
7697 if (PrivatizeDevicePointers) {
7699
7707 RCG(CGF);
7708 } else {
7709
7710
7711 std::optional CaptureRegion;
7712 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
7713
7715 for (const Expr *E : C->varlist()) {
7717 if (const auto *OED = dyn_cast(D))
7719 }
7721 for (const Expr *E : C->varlist()) {
7723 if (const auto *OED = dyn_cast(D))
7725 }
7726 } else {
7727 CaptureRegion = OMPD_unknown;
7728 }
7729
7730 OMPLexicalScope Scope(CGF, S, CaptureRegion);
7731 RCG(CGF);
7732 }
7733 };
7734
7735
7738
7739
7740
7741
7742 OMPLexicalScope Scope(CGF, S);
7744 PrivRCG);
7745 };
7746
7748
7749
7750
7751 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
7752 RCG(*this);
7753 return;
7754 }
7755
7756
7757 const Expr *IfCond = nullptr;
7758 if (const auto *C = S.getSingleClause<OMPIfClause>())
7759 IfCond = C->getCondition();
7760
7761
7765
7766
7768
7769
7770 CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
7771 Info);
7772}
7773
7776
7777
7778 if (CGM.getLangOpts().OMPTargetTriples.empty())
7779 return;
7780
7781
7782 const Expr *IfCond = nullptr;
7783 if (const auto *C = S.getSingleClause<OMPIfClause>())
7784 IfCond = C->getCondition();
7785
7786
7790
7791 OMPLexicalScope Scope(*this, S, OMPD_task);
7792 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
7793}
7794
7797
7798
7799 if (CGM.getLangOpts().OMPTargetTriples.empty())
7800 return;
7801
7802
7803 const Expr *IfCond = nullptr;
7804 if (const auto *C = S.getSingleClause<OMPIfClause>())
7805 IfCond = C->getCondition();
7806
7807
7811
7812 OMPLexicalScope Scope(*this, S, OMPD_task);
7813 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
7814}
7815
7819
7820 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
7821 Action.Enter(CGF);
7823 Action.Enter(CGF);
7831
7834 };
7839}
7840
7846 };
7847 llvm::Function *Fn;
7848 llvm::Constant *Addr;
7849
7850 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7851 S, ParentName, Fn, Addr, true, CodeGen);
7852 assert(Fn && Addr && "Target device function emission failed.");
7853}
7854
7862
7866 Action.Enter(CGF);
7867
7868
7870 Action.Enter(CGF);
7872 CGF, OMPD_target_parallel_for, S.hasCancel());
7875 };
7878}
7879
7883
7886 };
7887 llvm::Function *Fn;
7888 llvm::Constant *Addr;
7889
7890 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7891 S, ParentName, Fn, Addr, true, CodeGen);
7892 assert(Fn && Addr && "Target device function emission failed.");
7893}
7894
7902
7903static void
7907 Action.Enter(CGF);
7908
7909
7911 Action.Enter(CGF);
7914 };
7917}
7918
7922
7925 };
7926 llvm::Function *Fn;
7927 llvm::Constant *Addr;
7928
7929 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7930 S, ParentName, Fn, Addr, true, CodeGen);
7931 assert(Fn && Addr && "Target device function emission failed.");
7932}
7933
7941
7942
7949
7952
7953 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
7955 {
7956 OMPLexicalScope Scope(*this, S, OMPD_taskloop, false);
7958 }
7961 const Expr *IfCond = nullptr;
7962 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7963 if (C->getNameModifier() == OMPD_unknown ||
7964 C->getNameModifier() == OMPD_taskloop) {
7965 IfCond = C->getCondition();
7966 break;
7967 }
7968 }
7969
7971
7973
7974 Data.Tied = true;
7975
7977
7978 Data.Schedule.setInt(false);
7980 Data.HasModifier =
7981 (Clause->getModifier() == OMPC_GRAINSIZE_strict) ? true : false;
7982 } else if (const auto *Clause = S.getSingleClause<OMPNumTasksClause>()) {
7983
7984 Data.Schedule.setInt(true);
7986 Data.HasModifier =
7987 (Clause->getModifier() == OMPC_NUMTASKS_strict) ? true : false;
7988 }
7989
7990 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
7991
7992
7993
7994
7995
7996
7997
7998
7999
8000 bool CondConstant;
8001 llvm::BasicBlock *ContBlock = nullptr;
8002 OMPLoopScope PreInitScope(CGF, S);
8003 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
8004 if (!CondConstant)
8005 return;
8006 } else {
8007 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
8008 ContBlock = CGF.createBasicBlock("taskloop.if.end");
8009 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
8010 CGF.getProfileCount(&S));
8011 CGF.EmitBlock(ThenBlock);
8012 CGF.incrementProfileCounter(&S);
8013 }
8014
8015 (void)CGF.EmitOMPLinearClauseInit(S);
8016
8018
8019 enum { LowerBound = 5, UpperBound, Stride, LastIter };
8021 auto *LBP = std::next(I, LowerBound);
8022 auto *UBP = std::next(I, UpperBound);
8023 auto *STP = std::next(I, Stride);
8024 auto *LIP = std::next(I, LastIter);
8026 LoopScope);
8028 LoopScope);
8031 LoopScope);
8032 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8033 CGF.EmitOMPLinearClause(S, LoopScope);
8034 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
8036
8037 const Expr *IVExpr = S.getIterationVariable();
8039 CGF.EmitVarDecl(*IVDecl);
8040 CGF.EmitIgnoredExpr(S.getInit());
8041
8042
8043
8044
8045 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) {
8046 CGF.EmitVarDecl(*cast(LIExpr->getDecl()));
8047
8048 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
8049 }
8050
8051 {
8052 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, false);
8054 CGF, S,
8058 },
8059 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
8062 [&S](CodeGenFunction &CGF) {
8063 emitOMPLoopBodyWithStopPoint(CGF, S,
8064 CodeGenFunction::JumpDest());
8065 },
8066 [](CodeGenFunction &) {});
8067 });
8068 }
8069
8070 if (ContBlock) {
8071 CGF.EmitBranch(ContBlock);
8072 CGF.EmitBlock(ContBlock, true);
8073 }
8074
8075 if (HasLastprivateClause) {
8076 CGF.EmitOMPLastprivateClauseFinal(
8078 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
8079 CGF.GetAddrOfLocalVar(*LIP), false,
8080 (*LIP)->getType(), S.getBeginLoc())));
8081 }
8083 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {
8084 return CGF.Builder.CreateIsNotNull(
8086 (*LIP)->getType(), S.getBeginLoc()));
8087 });
8088 };
8089 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
8090 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
8092 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
8094 OMPLoopScope PreInitScope(CGF, S);
8095 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
8096 OutlinedFn, SharedsTy,
8097 CapturedStruct, IfCond, Data);
8098 };
8099 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
8101 };
8102 if (Data.Nogroup) {
8104 } else {
8105 CGM.getOpenMPRuntime().emitTaskgroupRegion(
8106 *this,
8107 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
8109 Action.Enter(CGF);
8112 },
8113 S.getBeginLoc());
8114 }
8115}
8116
8118 auto LPCRegion =
8121}
8122
8125 auto LPCRegion =
8127 OMPLexicalScope Scope(*this, S);
8129}
8130
8134 Action.Enter(CGF);
8136 };
8137 auto LPCRegion =
8139 OMPLexicalScope Scope(*this, S, std::nullopt, false);
8140 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
8141}
8142
8146 Action.Enter(CGF);
8148 };
8149 auto LPCRegion =
8151 OMPLexicalScope Scope(*this, S, std::nullopt, false);
8152 CGM.getOpenMPRuntime().emitMaskedRegion(*this, CodeGen, S.getBeginLoc());
8153}
8154
8158 Action.Enter(CGF);
8160 };
8161 auto LPCRegion =
8163 OMPLexicalScope Scope(*this, S);
8164 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
8165}
8166
8170 Action.Enter(CGF);
8172 };
8173 auto LPCRegion =
8175 OMPLexicalScope Scope(*this, S);
8176 CGM.getOpenMPRuntime().emitMaskedRegion(*this, CodeGen, S.getBeginLoc());
8177}
8178
8182 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8184 Action.Enter(CGF);
8186 };
8187 OMPLexicalScope Scope(CGF, S, OMPD_parallel, false);
8188 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
8189 S.getBeginLoc());
8190 };
8191 auto LPCRegion =
8195}
8196
8200 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8202 Action.Enter(CGF);
8204 };
8205 OMPLexicalScope Scope(CGF, S, OMPD_parallel, false);
8206 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,
8207 S.getBeginLoc());
8208 };
8209 auto LPCRegion =
8213}
8214
8218 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8220 Action.Enter(CGF);
8222 };
8223 OMPLexicalScope Scope(CGF, S, OMPD_parallel, false);
8224 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
8225 S.getBeginLoc());
8226 };
8227 auto LPCRegion =
8231}
8232
8236 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8238 Action.Enter(CGF);
8240 };
8241 OMPLexicalScope Scope(CGF, S, OMPD_parallel, false);
8242 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,
8243 S.getBeginLoc());
8244 };
8245 auto LPCRegion =
8249}
8250
8251
8254
8255
8256 if (CGM.getLangOpts().OMPTargetTriples.empty())
8257 return;
8258
8259
8260 const Expr *IfCond = nullptr;
8261 if (const auto *C = S.getSingleClause<OMPIfClause>())
8262 IfCond = C->getCondition();
8263
8264
8268
8269 OMPLexicalScope Scope(*this, S, OMPD_task);
8270 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
8271}
8272
8275
8276
8277
8279 if (const auto *C = S.getSingleClause<OMPBindClause>())
8280 BindKind = C->getBindKind();
8281
8282 switch (BindKind) {
8283 case OMPC_BIND_parallel:
8285 case OMPC_BIND_teams:
8287 case OMPC_BIND_thread:
8290 break;
8291 }
8292
8293
8295
8296 const Stmt *CS =
8298 const auto *ForS = dyn_cast(CS);
8305 } else {
8307 }
8308 };
8309 OMPLexicalScope Scope(*this, S, OMPD_unknown);
8310 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_loop, CodeGen);
8311}
8312
8315
8316
8318 Action.Enter(CGF);
8321 };
8322 {
8323 auto LPCRegion =
8327 }
8328
8330}
8331
8334
8335
8336 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8338 };
8339
8340
8341 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8343 Action.Enter(CGF);
8348 CodeGenDistribute);
8350 };
8353 [](CodeGenFunction &) { return nullptr; });
8354}
8355
8356#ifndef NDEBUG
8358 std::string StatusMsg,
8360 bool IsDevice = CGF.CGM.getLangOpts().OpenMPIsTargetDevice;
8361 if (IsDevice)
8362 StatusMsg += ": DEVICE";
8363 else
8364 StatusMsg += ": HOST";
8369 unsigned LineNo =
8370 PLoc.isValid() ? PLoc.getLine() : SM.getExpansionLineNumber(L);
8371 llvm::dbgs() << StatusMsg << ": " << FileName << ": " << LineNo << "\n";
8372}
8373#endif
8374
8378 Action.Enter(CGF);
8379
8380
8383 S.getDistInc());
8384 };
8385
8386
8387 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8389 Action.Enter(CGF);
8394 CGF, OMPD_distribute, CodeGenDistribute, false);
8396 };
8401 CodeGenTeams);
8404}
8405
8409 Action.Enter(CGF);
8410
8413 };
8414
8415
8418 Action.Enter(CGF);
8423 CGF, OMPD_distribute, CodeGenDistribute, false);
8425 };
8432}
8433
8444
8448
8452 else
8454 };
8455 llvm::Function *Fn;
8456 llvm::Constant *Addr;
8457
8458 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8459 S, ParentName, Fn, Addr, true, CodeGen);
8460 assert(Fn && Addr &&
8461 "Target device function emission failed for 'target teams loop'.");
8462}
8463
8467 Action.Enter(CGF);
8468
8470 Action.Enter(CGF);
8472 CGF, OMPD_target_parallel_loop, false);
8475 };
8478}
8479
8483
8486 };
8487 llvm::Function *Fn;
8488 llvm::Constant *Addr;
8489
8490 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8491 S, ParentName, Fn, Addr, true, CodeGen);
8492 assert(Fn && Addr && "Target device function emission failed.");
8493}
8494
8495
8496
8504
8507 if (const auto *SD = dyn_cast(&D)) {
8509 return;
8510 }
8511 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
8512 return;
8516
8518 for (const Expr *Ref : C->varlist()) {
8519 const auto *DRE = cast(Ref->IgnoreParenImpCasts());
8520 if (!DRE)
8521 continue;
8522 const auto *VD = dyn_cast(DRE->getDecl());
8524 continue;
8525 if (!CGF.LocalDeclMap.count(VD)) {
8528 }
8529 }
8530 }
8531 }
8536 } else {
8537 if (const auto *LD = dyn_cast(&D)) {
8538 for (const Expr *E : LD->counters()) {
8540 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
8543 }
8545
8546 if (!CGF.LocalDeclMap.count(VD))
8548 }
8549 }
8551 if (->getNumForLoops())
8552 continue;
8553 for (unsigned I = LD->getLoopsNumber(),
8554 E = C->getLoopNumIterations().size();
8555 I < E; ++I) {
8556 if (const auto *VD = dyn_cast(
8558
8559 if (!CGF.LocalDeclMap.count(VD))
8561 }
8562 }
8563 }
8564 }
8566 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());
8567 }
8568 };
8569 if (D.getDirectiveKind() == OMPD_atomic ||
8570 D.getDirectiveKind() == OMPD_critical ||
8571 D.getDirectiveKind() == OMPD_section ||
8572 D.getDirectiveKind() == OMPD_master ||
8573 D.getDirectiveKind() == OMPD_masked ||
8574 D.getDirectiveKind() == OMPD_unroll ||
8575 D.getDirectiveKind() == OMPD_assume) {
8576 EmitStmt(D.getAssociatedStmt());
8577 } else {
8578 auto LPCRegion =
8580 OMPSimdLexicalScope Scope(*this, D);
8581 CGM.getOpenMPRuntime().emitInlinedDirective(
8582 *this,
8584 : D.getDirectiveKind(),
8586 }
8587
8589}
8590
8592 EmitStmt(S.getAssociatedStmt());
8593}
Defines the clang::ASTContext interface.
static bool isAllocatableDecl(const VarDecl *VD)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
Definition CGStmtOpenMP.cpp:7031
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
Definition CGStmtOpenMP.cpp:2719
static const VarDecl * getBaseDecl(const Expr *Ref)
Definition CGStmtOpenMP.cpp:7604
static void emitTargetTeamsGenericLoopRegionAsParallel(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
Definition CGStmtOpenMP.cpp:8375
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *V, SourceLocation Loc)
Definition CGStmtOpenMP.cpp:6375
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
Definition CGStmtOpenMP.cpp:6640
static void emitScanBasedDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen, llvm::function_ref< void(CodeGenFunction &)> FirstGen, llvm::function_ref< void(CodeGenFunction &)> SecondGen)
Emits the code for the directive with inscan reductions.
Definition CGStmtOpenMP.cpp:3909
static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, RValue RVal)
Definition CGStmtOpenMP.cpp:6339
static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T)
Definition CGStmtOpenMP.cpp:5633
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV)
Definition CGStmtOpenMP.cpp:451
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars)
Definition CGStmtOpenMP.cpp:3408
static void emitScanBasedDirectiveFinals(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Copies final inscan reductions values to the original variables.
Definition CGStmtOpenMP.cpp:3845
static void checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Definition CGStmtOpenMP.cpp:1580
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
Definition CGStmtOpenMP.cpp:3762
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
Definition CGStmtOpenMP.cpp:7863
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
Definition CGStmtOpenMP.cpp:2675
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
Definition CGStmtOpenMP.cpp:6575
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
Definition CGStmtOpenMP.cpp:6300
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S, const OMPExecutableDirective &D)
Definition CGStmtOpenMP.cpp:6176
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
Definition CGStmtOpenMP.cpp:2429
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
Definition CGStmtOpenMP.cpp:6433
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Definition CGStmtOpenMP.cpp:3348
static void emitTargetTeamsGenericLoopRegionAsDistribute(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
Definition CGStmtOpenMP.cpp:8406
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
Definition CGStmtOpenMP.cpp:7816
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
Definition CGStmtOpenMP.cpp:3776
static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Definition CGStmtOpenMP.cpp:4563
static void emitRestoreIP(CodeGenFunction &CGF, const T *C, llvm::OpenMPIRBuilder::InsertPointTy AllocaIP, llvm::OpenMPIRBuilder &OMPBuilder)
Definition CGStmtOpenMP.cpp:6187
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
Definition CGStmtOpenMP.cpp:6943
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Definition CGStmtOpenMP.cpp:2502
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Definition CGStmtOpenMP.cpp:2352
static bool isSimdSupportedByOpenMPIRBuilder(const OMPLoopDirective &S)
Definition CGStmtOpenMP.cpp:2820
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Definition CGStmtOpenMP.cpp:1636
static void applyConservativeSimdOrderedDirective(const Stmt &AssociatedStmt, LoopInfoStack &LoopStack)
Definition CGStmtOpenMP.cpp:2590
static bool emitWorksharingDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, bool HasCancel)
Definition CGStmtOpenMP.cpp:4052
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Definition CGStmtOpenMP.cpp:1544
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
Definition CGStmtOpenMP.cpp:3282
static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF, std::string StatusMsg, const OMPExecutableDirective &D)
Definition CGStmtOpenMP.cpp:8357
static bool isForSupportedByOpenMPIRBuilder(const OMPLoopDirective &S, bool HasCancel)
Definition CGStmtOpenMP.cpp:4098
static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, SourceLocation Loc)
Definition CGStmtOpenMP.cpp:6347
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the 'for' loop has a dispatch schedule (e.g.
Definition CGStmtOpenMP.cpp:3391
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *UE, const Expr *D, const Expr *CE, bool IsXLHSInRHSPart, bool IsFailOnly, SourceLocation Loc)
Definition CGStmtOpenMP.cpp:6828
#define TTL_CODEGEN_TYPE
Definition CGStmtOpenMP.cpp:46
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
Definition CGStmtOpenMP.cpp:6313
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
Definition CGStmtOpenMP.cpp:5307
static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF, const CapturedStmt *S)
Emit a captured statement and return the function as well as its captured closure context.
Definition CGStmtOpenMP.cpp:2086
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
Definition CGStmtOpenMP.cpp:2667
static void emitOMPDistributeDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM)
Definition CGStmtOpenMP.cpp:6160
static void emitOMPCopyinClause(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Definition CGStmtOpenMP.cpp:1714
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
Definition CGStmtOpenMP.cpp:7408
static bool hasOrderedDirective(const Stmt *S)
Definition CGStmtOpenMP.cpp:2575
static llvm::CallInst * emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap, llvm::ArrayRef< llvm::Value * > Args)
Emit a call to a previously captured closure.
Definition CGStmtOpenMP.cpp:2100
static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Definition CGStmtOpenMP.cpp:4603
static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, int MaxLevel, int Level=0)
Definition CGStmtOpenMP.cpp:1974
static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM, bool HasCancel)
Definition CGStmtOpenMP.cpp:4150
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value * > &)
Definition CGStmtOpenMP.cpp:1710
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
Definition CGStmtOpenMP.cpp:7904
static void emitOMPSimdDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM)
Definition CGStmtOpenMP.cpp:2885
static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D, const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly, SourceLocation Loc)
Definition CGStmtOpenMP.cpp:6747
std::pair< llvm::Function *, llvm::Value * > EmittedClosureTy
Definition CGStmtOpenMP.cpp:2082
static OpenMPDirectiveKind getEffectiveDirectiveKind(const OMPExecutableDirective &S)
Definition CGStmtOpenMP.cpp:343
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
Definition CGStmtOpenMP.cpp:7110
static void buildDependences(const OMPExecutableDirective &S, OMPTaskDataTy &Data)
Definition CGStmtOpenMP.cpp:4885
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
Definition CGStmtOpenMP.cpp:6623
static void emitScanBasedDirectiveDecls(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Emits internal temp array declarations for the directive with inscan reductions.
Definition CGStmtOpenMP.cpp:3793
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
Definition CGStmtOpenMP.cpp:7459
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
Definition CGStmtOpenMP.cpp:7200
static llvm::MapVector< llvm::Value *, llvm::Value * > GetAlignedMapping(const OMPLoopDirective &S, CodeGenFunction &CGF)
Definition CGStmtOpenMP.cpp:2853
static llvm::omp::ScheduleKind convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind)
Definition CGStmtOpenMP.cpp:4130
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
Definition CGStmtOpenMP.cpp:7943
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Definition CGStmtOpenMP.cpp:7066
static void emitTargetParallelGenericLoopRegion(CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S, PrePostActionTy &Action)
Definition CGStmtOpenMP.cpp:8464
static QualType getCanonicalParamType(ASTContext &C, QualType T)
Definition CGStmtOpenMP.cpp:465
static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, const RegionCodeGenTy &SimdInitGen, const RegionCodeGenTy &BodyCodeGen)
Definition CGStmtOpenMP.cpp:2682
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
Definition CGStmtOpenMP.cpp:4230
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, SourceLocation Loc)
Definition CGStmtOpenMP.cpp:6406
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address > > &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value * > > &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
Definition CGStmtOpenMP.cpp:506
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
Definition CGStmtOpenMP.cpp:3426
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
Definition CGStmtOpenMP.cpp:7155
This file defines OpenMP nodes for declarative directives.
TokenType getType() const
Returns the token's type, e.g.
FormatToken * Next
The next token in the unwrapped line.
static const Decl * getCanonicalDecl(const Decl *D)
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the PrettyStackTraceEntry class, which is used to make crashes give more contextual informati...
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
This represents 'pragma omp cancel' directive.
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
This represents 'pragma omp cancellation point' directive.
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
The base class for all transformation directives of canonical loop sequences (currently only 'fuse')
This represents 'pragma omp distribute' directive.
This represents 'pragma omp distribute parallel for' composite directive.
This represents 'pragma omp distribute parallel for simd' composite directive.
This represents 'pragma omp distribute simd' composite directive.
This represents 'pragma omp error' directive.
Represents the 'pragma omp fuse' loop transformation directive.
Stmt * getTransformedStmt() const
Gets the associated loops after the transformation.
This represents 'pragma omp loop' directive.
Represents the 'pragma omp interchange' loop transformation directive.
Stmt * getTransformedStmt() const
Gets the associated loops after the transformation.
This represents 'pragma omp interop' directive.
This represents 'pragma omp masked' directive.
This represents 'pragma omp masked taskloop' directive.
This represents 'pragma omp masked taskloop simd' directive.
This represents 'pragma omp master taskloop' directive.
This represents 'pragma omp master taskloop simd' directive.
This represents 'pragma omp parallel masked taskloop' directive.
This represents 'pragma omp parallel masked taskloop simd' directive.
This represents 'pragma omp parallel master taskloop' directive.
This represents 'pragma omp parallel master taskloop simd' directive.
Represents the 'pragma omp reverse' loop transformation directive.
Stmt * getTransformedStmt() const
Gets/sets the associated loops after the transformation, i.e.
This represents 'pragma omp scan' directive.
This represents the 'pragma omp stripe' loop transformation directive.
Stmt * getTransformedStmt() const
Gets/sets the associated loops after striping.
This represents 'pragma omp target data' directive.
This represents 'pragma omp target' directive.
This represents 'pragma omp target enter data' directive.
This represents 'pragma omp target exit data' directive.
This represents 'pragma omp target parallel' directive.
This represents 'pragma omp target parallel for' directive.
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'pragma omp target parallel for simd' directive.
This represents 'pragma omp target parallel loop' directive.
This represents 'pragma omp target simd' directive.
This represents 'pragma omp target teams' directive.
This represents 'pragma omp target teams distribute' combined directive.
This represents 'pragma omp target teams distribute parallel for' combined directive.
This represents 'pragma omp target teams distribute parallel for simd' combined directive.
This represents 'pragma omp target teams distribute simd' combined directive.
This represents 'pragma omp target teams loop' directive.
bool canBeParallelFor() const
Return true if current loop directive's associated loop can be a parallel for.
This represents 'pragma omp target update' directive.
This represents 'pragma omp taskloop' directive.
This represents 'pragma omp taskloop simd' directive.
This represents 'pragma omp teams' directive.
This represents 'pragma omp teams distribute' directive.
This represents 'pragma omp teams distribute parallel for' composite directive.
This represents 'pragma omp teams distribute parallel for simd' composite directive.
This represents 'pragma omp teams distribute simd' combined directive.
This represents 'pragma omp teams loop' directive.
This represents the 'pragma omp tile' loop transformation directive.
Stmt * getTransformedStmt() const
Gets/sets the associated loops after tiling.
This represents the 'pragma omp unroll' loop transformation directive.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
TranslationUnitDecl * getTranslationUnitDecl() const
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location,...
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CanQualType getCanonicalTagType(const TagDecl *TD) const
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Represents an attribute applied to a statement.
ArrayRef< const Attr * > getAttrs() const
static BinaryOperator * Create(const ASTContext &C, Expr *lhs, Expr *rhs, Opcode opc, QualType ResTy, ExprValueKind VK, ExprObjectKind OK, SourceLocation opLoc, FPOptionsOverride FPFeatures)
Represents the body of a CapturedStmt, and serves as its DeclContext.
unsigned getContextParamPosition() const
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
ImplicitParamDecl * getParam(unsigned i) const
This captures a statement into a function.
SourceLocation getEndLoc() const LLVM_READONLY
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
SourceLocation getBeginLoc() const LLVM_READONLY
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
CharUnits - This is an opaque type for sizes expressed in character units.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
CGFunctionInfo - Class to encapsulate the information about a function definition.
Manages list of lastprivate conditional decls for the specified directive.
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Manages list of nontemporal decls for the specified directive.
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
Manages list of nontemporal decls for the specified directive.
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, SourceLocation SeverityLoc=SourceLocation(), const Expr *Message=nullptr, SourceLocation MessageLoc=SourceLocation())
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc)
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const
Translates the native parameter of outlined function if this is required for target.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself.
API for captured statement code generation.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for correct setting/restoring of CapturedStmtInfo.
LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const
void ForceCleanup()
Force the emission of cleanups now, instead of waiting until this object is destroyed.
RAII for preserving necessary info during inlined region body codegen.
Cleanup action for allocate support.
RAII for preserving necessary info during Outlined region body codegen.
Controls insertion of cancellation exit blocks in worksharing constructs.
Save/restore original map of previously emitted local vars in case when we need to duplicate emission...
The class used to assign some variables some temporarily addresses.
bool apply(CodeGenFunction &CGF)
Applies new addresses to the list of the variables.
void restore(CodeGenFunction &CGF)
Restores original addresses of the variables.
bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD, Address TempAddr)
Sets the address of the variable LocalVD to be TempAddr in function CGF.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
void restoreMap()
Restore all mapped variables w/o clean up.
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const VarDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Manages parent directive for scan directives.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
void ForceCleanup(std::initializer_list< llvm::Value ** > ValuesToReload={})
Force the emission of cleanups now, instead of waiting until this object is destroyed.
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void EmitOMPParallelMaskedTaskLoopDirective(const OMPParallelMaskedTaskLoopDirective &S)
Definition CGStmtOpenMP.cpp:8197
void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S)
Definition CGStmtOpenMP.cpp:4803
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
Definition CGStmtOpenMP.cpp:5606
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr, const VarDecl *ConditionalDecl=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
Definition CGStmtOpenMP.cpp:1197
void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data, CodeGenFunction &CGF, const CapturedStmt *CS, OMPPrivateScope &Scope)
Definition CGStmtOpenMP.cpp:5485
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
Definition CGStmtOpenMP.cpp:7950
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
Definition CGStmtOpenMP.cpp:6357
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
Definition CGStmtOpenMP.cpp:7841
void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S)
Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
Definition CGStmtOpenMP.cpp:2138
void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S)
Definition CGStmtOpenMP.cpp:8273
void EmitOMPScanDirective(const OMPScanDirective &S)
Definition CGStmtOpenMP.cpp:5728
static bool hasScalarEvaluationKind(QualType T)
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * >(CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
LValue InitCapturedStruct(const CapturedStmt &S)
CGCapturedStmtInfo * CapturedStmtInfo
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
Definition CGStmtOpenMP.cpp:6170
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
Definition CGStmtOpenMP.cpp:4708
void EmitOMPMasterDirective(const OMPMasterDirective &S)
Definition CGStmtOpenMP.cpp:4571
void EmitOMPParallelMasterTaskLoopSimdDirective(const OMPParallelMasterTaskLoopSimdDirective &S)
Definition CGStmtOpenMP.cpp:8215
void EmitOMPSimdInit(const OMPLoopDirective &D)
Helpers for the OpenMP loop directives.
Definition CGStmtOpenMP.cpp:2601
const OMPExecutableDirective * OMPParentLoopDirectiveForScan
Parent loop-based directive for scan directive.
void EmitOMPFlushDirective(const OMPFlushDirective &S)
Definition CGStmtOpenMP.cpp:5688
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
Definition CGStmtOpenMP.cpp:7045
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Definition CGStmtOpenMP.cpp:893
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
Definition CGStmtOpenMP.cpp:5637
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Definition CGStmtOpenMP.cpp:7503
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
Definition CGStmtOpenMP.cpp:7132
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope, bool ForInscan=false)
Emit initial code for reduction variables.
Definition CGStmtOpenMP.cpp:1288
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
Definition CGStmtOpenMP.cpp:3475
void EmitAutoVarDecl(const VarDecl &D)
EmitAutoVarDecl - Emit an auto variable declaration.
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
Definition CGStmtOpenMP.cpp:7178
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
Definition CGStmtOpenMP.cpp:5625
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
Definition CGStmtOpenMP.cpp:7895
const LangOptions & getLangOpts() const
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
Definition CGStmtOpenMP.cpp:1498
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
Definition CGStmtOpenMP.cpp:2013
void EmitOMPScopeDirective(const OMPScopeDirective &S)
Definition CGStmtOpenMP.cpp:4386
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
CodeGenFunction * ParentCGF
JumpDest ReturnBlock
ReturnBlock - Unified return block.
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
Definition CGStmtOpenMP.cpp:7238
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location)
Converts Location to a DebugLoc, if debug information is enabled.
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
Definition CGStmtOpenMP.cpp:1062
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
Definition CGStmtOpenMP.cpp:2467
llvm::BasicBlock * OMPBeforeScanBlock
void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S)
Definition CGStmtOpenMP.cpp:2978
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
Definition CGStmtOpenMP.cpp:2387
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
Definition CGStmtOpenMP.cpp:404
void EmitOMPDepobjDirective(const OMPDepobjDirective &S)
Definition CGStmtOpenMP.cpp:5703
void EmitOMPMetaDirective(const OMPMetaDirective &S)
Definition CGStmtOpenMP.cpp:1931
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
Definition CGStmtOpenMP.cpp:4653
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
Definition CGStmtOpenMP.cpp:8117
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
void EmitOMPCancelDirective(const OMPCancelDirective &S)
Definition CGStmtOpenMP.cpp:7517
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
Definition CGStmtOpenMP.cpp:5621
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type,...
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
Definition CGStmtOpenMP.cpp:6210
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
Definition CGStmtOpenMP.cpp:3518
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
Definition CGStmtOpenMP.cpp:368
llvm::CanonicalLoopInfo * EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth)
Emit the Stmt S and return its topmost canonical loop, if any.
Definition CGStmtOpenMP.cpp:2112
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
Definition CGStmtOpenMP.cpp:4412
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPInteropDirective(const OMPInteropDirective &S)
Definition CGStmtOpenMP.cpp:7338
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
Definition CGStmtOpenMP.cpp:4830
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
Definition CGStmtOpenMP.cpp:7855
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
Definition CGStmtOpenMP.cpp:856
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
Definition CGStmtOpenMP.cpp:7548
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
Definition CGStmtOpenMP.cpp:7934
void EmitOMPTargetParallelGenericLoopDirective(const OMPTargetParallelGenericLoopDirective &S)
Emit combined directive 'target parallel loop' as if its constituent constructs are 'target',...
Definition CGStmtOpenMP.cpp:8497
void EmitOMPUseDeviceAddrClause(const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
Definition CGStmtOpenMP.cpp:7613
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
Definition CGStmtOpenMP.cpp:7314
void EmitOMPMaskedDirective(const OMPMaskedDirective &S)
Definition CGStmtOpenMP.cpp:4615
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
Definition CGStmtOpenMP.cpp:791
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
Definition CGStmtOpenMP.cpp:7269
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
CGDebugInfo * getDebugInfo()
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
Definition CGStmtOpenMP.cpp:5929
void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S)
Definition CGStmtOpenMP.cpp:8131
void EmitOMPReverseDirective(const OMPReverseDirective &S)
Definition CGStmtOpenMP.cpp:2972
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
Definition CGStmtOpenMP.cpp:384
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
Definition CGStmtOpenMP.cpp:7511
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
Definition CGStmtOpenMP.cpp:7451
void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S)
Definition CGStmtOpenMP.cpp:8143
llvm::BasicBlock * OMPScanDispatch
llvm::function_ref< std::pair< LValue, LValue >(CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
Definition CGStmtOpenMP.cpp:7795
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
Definition CGStmtOpenMP.cpp:7774
void EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S)
Definition CGStmtOpenMP.cpp:8167
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
Definition CGStmtOpenMP.cpp:6551
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
void EmitOMPParallelDirective(const OMPParallelDirective &S)
Definition CGStmtOpenMP.cpp:1846
void EmitOMPTaskDirective(const OMPTaskDirective &S)
Definition CGStmtOpenMP.cpp:5573
void EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S)
Definition CGStmtOpenMP.cpp:8155
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
Definition CGStmtOpenMP.cpp:3455
void EmitOMPAssumeDirective(const OMPAssumeDirective &S)
Definition CGStmtOpenMP.cpp:8591
int ExpectedOMPLoopDepth
Number of nested loop to be consumed by the last surrounding loop-associated directive.
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Definition CGStmtOpenMP.cpp:1037
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
Definition CGStmtOpenMP.cpp:7246
ASTContext & getContext() const
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
Definition CGStmtOpenMP.cpp:8252
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
void EmitOMPTargetTeamsGenericLoopDirective(const OMPTargetTeamsGenericLoopDirective &S)
Definition CGStmtOpenMP.cpp:8434
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitAutoVarCleanups(const AutoVarEmission &emission)
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
SmallVector< llvm::CanonicalLoopInfo *, 4 > OMPLoopNestStack
List of recently emitted OMPCanonicalLoops.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
Definition CGStmtOpenMP.cpp:7291
llvm::AtomicRMWInst * emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Order=llvm::AtomicOrdering::SequentiallyConsistent, llvm::SyncScope::ID SSID=llvm::SyncScope::System, const AtomicExpr *AE=nullptr)
Emit an atomicrmw instruction, and applying relevant metadata when applicable.
void EmitOMPFuseDirective(const OMPFuseDirective &S)
Definition CGStmtOpenMP.cpp:2985
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
Definition CGStmtOpenMP.cpp:7192
void EmitOMPUseDevicePtrClause(const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
Definition CGStmtOpenMP.cpp:7562
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
void EmitStmt(const Stmt *S, ArrayRef< const Attr * > Attrs={})
EmitStmt - Emit the code for the statement.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
Definition CGStmtOpenMP.cpp:4742
llvm::Type * ConvertTypeForMem(QualType T)
void EmitOMPInnerLoop(const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
Definition CGStmtOpenMP.cpp:2215
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
Definition CGStmtOpenMP.cpp:8123
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
Definition CGStmtOpenMP.cpp:7435
void EmitOMPTargetDirective(const OMPTargetDirective &S)
Definition CGStmtOpenMP.cpp:7059
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
Definition CGStmtOpenMP.cpp:7919
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::BasicBlock * OMPScanExitBlock
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
Definition CGStmtOpenMP.cpp:7093
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
Definition CGStmtOpenMP.cpp:8505
void EmitOMPErrorDirective(const OMPErrorDirective &S)
Definition CGStmtOpenMP.cpp:5611
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
Definition CGStmtOpenMP.cpp:5336
void EmitOMPParallelMaskedTaskLoopSimdDirective(const OMPParallelMaskedTaskLoopSimdDirective &S)
Definition CGStmtOpenMP.cpp:8233
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
Definition CGStmtOpenMP.cpp:7146
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
Definition CGStmtOpenMP.cpp:7662
Address GenerateCapturedStmtArgument(const CapturedStmt &S)
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
Definition CGStmtOpenMP.cpp:1134
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
Definition CGStmtOpenMP.cpp:7487
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, const OMPExecutableDirective &D)
Definition CGStmtOpenMP.cpp:694
void EmitOMPSimdDirective(const OMPSimdDirective &S)
Definition CGStmtOpenMP.cpp:2956
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S)
Definition CGStmtOpenMP.cpp:8313
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
Definition CGStmtOpenMP.cpp:3498
void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S)
Definition CGStmtOpenMP.cpp:8332
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
Definition CGStmtOpenMP.cpp:2276
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
Definition CGStmtOpenMP.cpp:8480
void EmitOMPUnrollDirective(const OMPUnrollDirective &S)
Definition CGStmtOpenMP.cpp:2991
void EmitOMPStripeDirective(const OMPStripeDirective &S)
Definition CGStmtOpenMP.cpp:2966
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
Definition CGStmtOpenMP.cpp:4518
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
Definition CGStmtOpenMP.cpp:3484
llvm::Function * GenerateCapturedStmtFunction(const CapturedStmt &S)
Creates the outlined function for a CapturedStmt.
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
Definition CGStmtOpenMP.cpp:7880
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
Definition CGStmtOpenMP.cpp:8445
void EmitOMPTileDirective(const OMPTileDirective &S)
Definition CGStmtOpenMP.cpp:2960
void EmitDecl(const Decl &D, bool EvaluateConditionDecl=false)
EmitDecl - Emit a declaration.
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
Definition CGStmtOpenMP.cpp:6860
llvm::BasicBlock * OMPAfterScanBlock
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr)
Try to emit a reference to the given value without producing it as an l-value.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
void EmitOMPParallelMasterTaskLoopDirective(const OMPParallelMasterTaskLoopDirective &S)
Definition CGStmtOpenMP.cpp:8179
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
Definition CGStmtOpenMP.cpp:3465
void EmitOMPSectionDirective(const OMPSectionDirective &S)
Definition CGStmtOpenMP.cpp:4487
void EnsureInsertPoint()
EnsureInsertPoint - Ensure that an insertion point is defined so that emitted IR has a place to go.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
Definition CGStmtOpenMP.cpp:4210
llvm::LLVMContext & getLLVMContext()
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
Definition CGStmtOpenMP.cpp:7224
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S)
Definition CGStmtOpenMP.cpp:4776
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
Definition CGStmtOpenMP.cpp:4919
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPForDirective(const OMPForDirective &S)
Definition CGStmtOpenMP.cpp:4206
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
Definition CGStmtOpenMP.cpp:2314
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Definition CGStmtOpenMP.cpp:2622
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CodeGenTypes & getTypes()
const llvm::DataLayout & getDataLayout() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGFunctionInfo & arrangeDeviceKernelCallerDeclaration(QualType resultType, const FunctionArgList &args)
A device kernel caller function is an offload device entry point function with a target device depend...
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
llvm::Value * getPointer(CodeGenFunction &CGF) const
Address getAddress() const
void setAddress(Address address)
A stack of loop information corresponding to loop nesting levels.
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
An abstract representation of an aligned address.
llvm::PointerType * getType() const
Return the type of the pointer value.
llvm::Value * getPointer() const
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void setAction(PrePostActionTy &Action) const
Complex values, per C99 6.2.5p11.
CompoundStmt - This represents a group of statements like { stmt stmt }.
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext * getParent()
getParent - Returns the containing DeclContext.
A reference to a declared variable, function, enum, etc.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr, NonOdrUseReason NOUR=NOUR_None)
DeclStmt - Adaptor class for mixing declarations with statements and expressions.
Decl - This represents one declaration (or definition), e.g.
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
SourceLocation getLocation() const
The name of a declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
Expr * IgnoreImplicitAsWritten() LLVM_READONLY
Skip past any implicit AST nodes which might surround this expression until reaching a fixed point.
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Represents a function declaration or definition.
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool UsesFPIntrin=false, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=ConstexprSpecKind::Unspecified, const AssociatedConstraint &TrailingRequiresClause={})
GlobalDecl - represents a global declaration.
One of these records is kept for each identifier that is lexed.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat, FPOptionsOverride FPO)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Represents a point when we exit a loop.
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
A C++ nested-name-specifier augmented with source location information.
This represents 'acq_rel' clause in the 'pragma omp atomic|flush' directives.
This represents 'acquire' clause in the 'pragma omp atomic|flush' directives.
This represents clause 'aligned' in the 'pragma omp ...' directives.
This represents 'bind' clause in the 'pragma omp ...' directives.
static OMPClauseWithPreInit * get(OMPClause *C)
This is a basic class for representing single OpenMP clause.
This represents clause 'copyin' in the 'pragma omp ...' directives.
This represents clause 'copyprivate' in the 'pragma omp ...' directives.
This represents implicit clause 'depend' for the 'pragma omp task' directive.
This represents implicit clause 'depobj' for the 'pragma omp depobj' directive.
This represents 'destroy' clause in the 'pragma omp depobj' directive or the 'pragma omp interop' dir...
This represents 'device' clause in the 'pragma omp ...' directive.
This represents 'dist_schedule' clause in the 'pragma omp ...' directive.
This represents the 'doacross' clause for the 'pragma omp ordered' directive.
This represents 'fail' clause in the 'pragma omp atomic' directive.
This represents 'filter' clause in the 'pragma omp ...' directive.
This represents 'final' clause in the 'pragma omp ...' directive.
This represents clause 'firstprivate' in the 'pragma omp ...' directives.
This represents implicit clause 'flush' for the 'pragma omp flush' directive.
Representation of the 'full' clause of the 'pragma omp unroll' directive.
This represents 'grainsize' clause in the 'pragma omp ...' directive.
This represents 'hint' clause in the 'pragma omp ...' directive.
This represents 'if' clause in the 'pragma omp ...' directive.
This represents clause 'in_reduction' in the 'pragma omp task' directives.
This represents clause 'inclusive' in the 'pragma omp scan' directive.
This represents the 'init' clause in 'pragma omp ...' directives.
This represents clause 'lastprivate' in the 'pragma omp ...' directives.
This represents clause 'linear' in the 'pragma omp ...' directives.
This represents the 'message' clause in the 'pragma omp error' and the 'pragma omp parallel' directiv...
Expr * getMessageString() const
Returns message string of the clause.
This represents 'nogroup' clause in the 'pragma omp ...' directive.
This represents 'nowait' clause in the 'pragma omp ...' directive.
This represents 'num_tasks' clause in the 'pragma omp ...' directive.
This represents 'num_teams' clause in the 'pragma omp ...' directive.
This represents 'num_threads' clause in the 'pragma omp ...' directive.
This represents 'order' clause in the 'pragma omp ...' directive.
This represents 'ordered' clause in the 'pragma omp ...' directive.
Representation of the 'partial' clause of the 'pragma omp unroll' directive.
This represents 'priority' clause in the 'pragma omp ...' directive.
This represents clause 'private' in the 'pragma omp ...' directives.
This represents 'proc_bind' clause in the 'pragma omp ...' directive.
This represents clause 'reduction' in the 'pragma omp ...' directives.
This represents 'relaxed' clause in the 'pragma omp atomic' directives.
This represents 'release' clause in the 'pragma omp atomic|flush' directives.
This represents 'simd' clause in the 'pragma omp ...' directive.
This represents 'safelen' clause in the 'pragma omp ...' directive.
This represents 'schedule' clause in the 'pragma omp ...' directive.
This represents 'seq_cst' clause in the 'pragma omp atomic|flush' directives.
This represents the 'severity' clause in the 'pragma omp error' and the 'pragma omp parallel' directi...
OpenMPSeverityClauseKind getSeverityKind() const
Returns kind of the clause.
This represents 'simdlen' clause in the 'pragma omp ...' directive.
This represents clause 'task_reduction' in the 'pragma omp taskgroup' directives.
This represents 'thread_limit' clause in the 'pragma omp ...' directive.
This represents 'untied' clause in the 'pragma omp ...' directive.
This represents 'update' clause in the 'pragma omp atomic' directive.
This represents the 'use' clause in 'pragma omp ...' directives.
This represents clause 'use_device_addr' in the 'pragma omp ...' directives.
This represents clause 'use_device_ptr' in the 'pragma omp ...' directives.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
PointerType - C99 6.7.5.1 - Pointer Declarators.
Represents an unpacked "presumed" location which can be presented to the user.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
If a crash happens while one of these objects are live, the message is printed out along with the spe...
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Represents a struct/union/class.
field_range fields() const
field_iterator field_begin() const
Base for LValueReferenceType and RValueReferenceType.
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
A trivial tuple used to represent a source range.
SourceLocation getEnd() const
SourceLocation getBegin() const
Stmt - This represents one statement.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
SourceLocation getBeginLoc() const LLVM_READONLY
bool isPointerType() const
const T * castAs() const
Member-template castAs.
bool isReferenceType() const
bool isLValueReferenceType() const
bool isAnyComplexType() const
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
static UnaryOperator * Create(const ASTContext &C, Expr *input, Opcode opc, QualType type, ExprValueKind VK, ExprObjectKind OK, SourceLocation l, bool CanOverflow, FPOptionsOverride FPFeatures)
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
TLSKind getTLSKind() const
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
@ CInit
C-style initialization with assignment.
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
const Expr * getInit() const
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
@ TLS_None
Not a TLS variable.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
bool Inc(InterpState &S, CodePtr OpPC, bool CanOverflow)
- Pops a pointer from the stack 2) Load the value from the pointer 3) Writes the value increased by ...
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CanQual< Type > CanQualType
Represents a canonical, potentially-qualified type.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
bool isa(CodeGen::Address addr)
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
@ OK_Ordinary
An ordinary object is located at an address in memory.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ Tile
'tile' clause, allowed on 'loop' and Combined constructs.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_unknown
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
@ OMPC_DIST_SCHEDULE_unknown
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
bool isOpenMPGenericLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive constitutes a 'loop' directive in the outermost nest.
OpenMPBindClauseKind
OpenMP bindings for the 'bind' clause.
const FunctionProtoType * T
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
bool IsXLHSInRHSPart
True if UE has the first form and false if the second.
bool IsPostfixUpdate
True if original value of 'x' must be stored in 'v', not an updated one.
@ Dtor_Complete
Complete object dtor.
OpenMPSeverityClauseKind
OpenMP attributes for 'severity' clause.
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
OpenMPNumThreadsClauseModifier
@ OMPC_NUMTHREADS_unknown
bool IsFailOnly
True if 'v' is updated only when the condition is false (compare capture only).
U cast(CodeGen::Address addr)
llvm::omp::Clause OpenMPClauseKind
OpenMP clauses.
@ ThreadPrivateVar
Parameter for Thread private variable.
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
Struct with the values to be passed to the static runtime function.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::BasicBlock * getBlock() const
static Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
Definition CGStmtOpenMP.cpp:1777
llvm::OpenMPIRBuilder::InsertPointTy InsertPointTy
static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region that will be outlined in OpenMPIRBuilder::finalize().
Definition CGStmtOpenMP.cpp:1829
static Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable /p VD.
Definition CGStmtOpenMP.cpp:1727
static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP, llvm::BasicBlock &FiniBB, llvm::Function *Fn, ArrayRef< llvm::Value * > Args)
static std::string getNameWithSeparators(ArrayRef< StringRef > Parts, StringRef FirstSeparator=".", StringRef Separator=".")
Get the platform-specific name separator.
Definition CGStmtOpenMP.cpp:1800
static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP)
Emit the Finalization for an OMP region.
static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region.
Definition CGStmtOpenMP.cpp:1812
unsigned NumberOfTargetItems
Address BasePointersArray
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::IntegerType * SizeTy
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Scheduling data for loop-based OpenMP directives.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule