clang: lib/CodeGen/TargetBuiltins/ARM.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
13#include "ABIInfo.h"
16#include "TargetInfo.h"
18#include "llvm/IR/InlineAsm.h"
19#include "llvm/IR/IntrinsicsAArch64.h"
20#include "llvm/IR/IntrinsicsARM.h"
21#include "llvm/IR/IntrinsicsBPF.h"
22#include "llvm/TargetParser/AArch64TargetParser.h"
23
24#include
25
26using namespace clang;
28using namespace llvm;
29
30static std::optionalCodeGenFunction::MSVCIntrin
33 switch (BuiltinID) {
34 default:
35 return std::nullopt;
36 case clang::AArch64::BI_BitScanForward:
37 case clang::AArch64::BI_BitScanForward64:
38 return MSVCIntrin::_BitScanForward;
39 case clang::AArch64::BI_BitScanReverse:
40 case clang::AArch64::BI_BitScanReverse64:
41 return MSVCIntrin::_BitScanReverse;
42 case clang::AArch64::BI_InterlockedAnd64:
43 return MSVCIntrin::_InterlockedAnd;
44 case clang::AArch64::BI_InterlockedExchange64:
45 return MSVCIntrin::_InterlockedExchange;
46 case clang::AArch64::BI_InterlockedExchangeAdd64:
47 return MSVCIntrin::_InterlockedExchangeAdd;
48 case clang::AArch64::BI_InterlockedExchangeSub64:
49 return MSVCIntrin::_InterlockedExchangeSub;
50 case clang::AArch64::BI_InterlockedOr64:
51 return MSVCIntrin::_InterlockedOr;
52 case clang::AArch64::BI_InterlockedXor64:
53 return MSVCIntrin::_InterlockedXor;
54 case clang::AArch64::BI_InterlockedDecrement64:
55 return MSVCIntrin::_InterlockedDecrement;
56 case clang::AArch64::BI_InterlockedIncrement64:
57 return MSVCIntrin::_InterlockedIncrement;
58 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
59 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
60 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
61 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
62 return MSVCIntrin::_InterlockedExchangeAdd_acq;
63 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
64 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
65 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
66 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
67 return MSVCIntrin::_InterlockedExchangeAdd_rel;
68 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
69 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
70 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
71 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
72 return MSVCIntrin::_InterlockedExchangeAdd_nf;
73 case clang::AArch64::BI_InterlockedExchange8_acq:
74 case clang::AArch64::BI_InterlockedExchange16_acq:
75 case clang::AArch64::BI_InterlockedExchange_acq:
76 case clang::AArch64::BI_InterlockedExchange64_acq:
77 case clang::AArch64::BI_InterlockedExchangePointer_acq:
78 return MSVCIntrin::_InterlockedExchange_acq;
79 case clang::AArch64::BI_InterlockedExchange8_rel:
80 case clang::AArch64::BI_InterlockedExchange16_rel:
81 case clang::AArch64::BI_InterlockedExchange_rel:
82 case clang::AArch64::BI_InterlockedExchange64_rel:
83 case clang::AArch64::BI_InterlockedExchangePointer_rel:
84 return MSVCIntrin::_InterlockedExchange_rel;
85 case clang::AArch64::BI_InterlockedExchange8_nf:
86 case clang::AArch64::BI_InterlockedExchange16_nf:
87 case clang::AArch64::BI_InterlockedExchange_nf:
88 case clang::AArch64::BI_InterlockedExchange64_nf:
89 case clang::AArch64::BI_InterlockedExchangePointer_nf:
90 return MSVCIntrin::_InterlockedExchange_nf;
91 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
92 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
93 case clang::AArch64::BI_InterlockedCompareExchange_acq:
94 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
95 case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
96 return MSVCIntrin::_InterlockedCompareExchange_acq;
97 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
98 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
99 case clang::AArch64::BI_InterlockedCompareExchange_rel:
100 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
101 case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
102 return MSVCIntrin::_InterlockedCompareExchange_rel;
103 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
104 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
105 case clang::AArch64::BI_InterlockedCompareExchange_nf:
106 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
107 return MSVCIntrin::_InterlockedCompareExchange_nf;
108 case clang::AArch64::BI_InterlockedCompareExchange128:
109 return MSVCIntrin::_InterlockedCompareExchange128;
110 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
111 return MSVCIntrin::_InterlockedCompareExchange128_acq;
112 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
113 return MSVCIntrin::_InterlockedCompareExchange128_nf;
114 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
115 return MSVCIntrin::_InterlockedCompareExchange128_rel;
116 case clang::AArch64::BI_InterlockedOr8_acq:
117 case clang::AArch64::BI_InterlockedOr16_acq:
118 case clang::AArch64::BI_InterlockedOr_acq:
119 case clang::AArch64::BI_InterlockedOr64_acq:
120 return MSVCIntrin::_InterlockedOr_acq;
121 case clang::AArch64::BI_InterlockedOr8_rel:
122 case clang::AArch64::BI_InterlockedOr16_rel:
123 case clang::AArch64::BI_InterlockedOr_rel:
124 case clang::AArch64::BI_InterlockedOr64_rel:
125 return MSVCIntrin::_InterlockedOr_rel;
126 case clang::AArch64::BI_InterlockedOr8_nf:
127 case clang::AArch64::BI_InterlockedOr16_nf:
128 case clang::AArch64::BI_InterlockedOr_nf:
129 case clang::AArch64::BI_InterlockedOr64_nf:
130 return MSVCIntrin::_InterlockedOr_nf;
131 case clang::AArch64::BI_InterlockedXor8_acq:
132 case clang::AArch64::BI_InterlockedXor16_acq:
133 case clang::AArch64::BI_InterlockedXor_acq:
134 case clang::AArch64::BI_InterlockedXor64_acq:
135 return MSVCIntrin::_InterlockedXor_acq;
136 case clang::AArch64::BI_InterlockedXor8_rel:
137 case clang::AArch64::BI_InterlockedXor16_rel:
138 case clang::AArch64::BI_InterlockedXor_rel:
139 case clang::AArch64::BI_InterlockedXor64_rel:
140 return MSVCIntrin::_InterlockedXor_rel;
141 case clang::AArch64::BI_InterlockedXor8_nf:
142 case clang::AArch64::BI_InterlockedXor16_nf:
143 case clang::AArch64::BI_InterlockedXor_nf:
144 case clang::AArch64::BI_InterlockedXor64_nf:
145 return MSVCIntrin::_InterlockedXor_nf;
146 case clang::AArch64::BI_InterlockedAnd8_acq:
147 case clang::AArch64::BI_InterlockedAnd16_acq:
148 case clang::AArch64::BI_InterlockedAnd_acq:
149 case clang::AArch64::BI_InterlockedAnd64_acq:
150 return MSVCIntrin::_InterlockedAnd_acq;
151 case clang::AArch64::BI_InterlockedAnd8_rel:
152 case clang::AArch64::BI_InterlockedAnd16_rel:
153 case clang::AArch64::BI_InterlockedAnd_rel:
154 case clang::AArch64::BI_InterlockedAnd64_rel:
155 return MSVCIntrin::_InterlockedAnd_rel;
156 case clang::AArch64::BI_InterlockedAnd8_nf:
157 case clang::AArch64::BI_InterlockedAnd16_nf:
158 case clang::AArch64::BI_InterlockedAnd_nf:
159 case clang::AArch64::BI_InterlockedAnd64_nf:
160 return MSVCIntrin::_InterlockedAnd_nf;
161 case clang::AArch64::BI_InterlockedIncrement16_acq:
162 case clang::AArch64::BI_InterlockedIncrement_acq:
163 case clang::AArch64::BI_InterlockedIncrement64_acq:
164 return MSVCIntrin::_InterlockedIncrement_acq;
165 case clang::AArch64::BI_InterlockedIncrement16_rel:
166 case clang::AArch64::BI_InterlockedIncrement_rel:
167 case clang::AArch64::BI_InterlockedIncrement64_rel:
168 return MSVCIntrin::_InterlockedIncrement_rel;
169 case clang::AArch64::BI_InterlockedIncrement16_nf:
170 case clang::AArch64::BI_InterlockedIncrement_nf:
171 case clang::AArch64::BI_InterlockedIncrement64_nf:
172 return MSVCIntrin::_InterlockedIncrement_nf;
173 case clang::AArch64::BI_InterlockedDecrement16_acq:
174 case clang::AArch64::BI_InterlockedDecrement_acq:
175 case clang::AArch64::BI_InterlockedDecrement64_acq:
176 return MSVCIntrin::_InterlockedDecrement_acq;
177 case clang::AArch64::BI_InterlockedDecrement16_rel:
178 case clang::AArch64::BI_InterlockedDecrement_rel:
179 case clang::AArch64::BI_InterlockedDecrement64_rel:
180 return MSVCIntrin::_InterlockedDecrement_rel;
181 case clang::AArch64::BI_InterlockedDecrement16_nf:
182 case clang::AArch64::BI_InterlockedDecrement_nf:
183 case clang::AArch64::BI_InterlockedDecrement64_nf:
184 return MSVCIntrin::_InterlockedDecrement_nf;
185 }
186 llvm_unreachable("must return from switch");
187}
188
189static std::optionalCodeGenFunction::MSVCIntrin
192 switch (BuiltinID) {
193 default:
194 return std::nullopt;
195 case clang::ARM::BI_BitScanForward:
196 case clang::ARM::BI_BitScanForward64:
197 return MSVCIntrin::_BitScanForward;
198 case clang::ARM::BI_BitScanReverse:
199 case clang::ARM::BI_BitScanReverse64:
200 return MSVCIntrin::_BitScanReverse;
201 case clang::ARM::BI_InterlockedAnd64:
202 return MSVCIntrin::_InterlockedAnd;
203 case clang::ARM::BI_InterlockedExchange64:
204 return MSVCIntrin::_InterlockedExchange;
205 case clang::ARM::BI_InterlockedExchangeAdd64:
206 return MSVCIntrin::_InterlockedExchangeAdd;
207 case clang::ARM::BI_InterlockedExchangeSub64:
208 return MSVCIntrin::_InterlockedExchangeSub;
209 case clang::ARM::BI_InterlockedOr64:
210 return MSVCIntrin::_InterlockedOr;
211 case clang::ARM::BI_InterlockedXor64:
212 return MSVCIntrin::_InterlockedXor;
213 case clang::ARM::BI_InterlockedDecrement64:
214 return MSVCIntrin::_InterlockedDecrement;
215 case clang::ARM::BI_InterlockedIncrement64:
216 return MSVCIntrin::_InterlockedIncrement;
217 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
218 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
219 case clang::ARM::BI_InterlockedExchangeAdd_acq:
220 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
221 return MSVCIntrin::_InterlockedExchangeAdd_acq;
222 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
223 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
224 case clang::ARM::BI_InterlockedExchangeAdd_rel:
225 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
226 return MSVCIntrin::_InterlockedExchangeAdd_rel;
227 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
228 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
229 case clang::ARM::BI_InterlockedExchangeAdd_nf:
230 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
231 return MSVCIntrin::_InterlockedExchangeAdd_nf;
232 case clang::ARM::BI_InterlockedExchange8_acq:
233 case clang::ARM::BI_InterlockedExchange16_acq:
234 case clang::ARM::BI_InterlockedExchange_acq:
235 case clang::ARM::BI_InterlockedExchange64_acq:
236 case clang::ARM::BI_InterlockedExchangePointer_acq:
237 return MSVCIntrin::_InterlockedExchange_acq;
238 case clang::ARM::BI_InterlockedExchange8_rel:
239 case clang::ARM::BI_InterlockedExchange16_rel:
240 case clang::ARM::BI_InterlockedExchange_rel:
241 case clang::ARM::BI_InterlockedExchange64_rel:
242 case clang::ARM::BI_InterlockedExchangePointer_rel:
243 return MSVCIntrin::_InterlockedExchange_rel;
244 case clang::ARM::BI_InterlockedExchange8_nf:
245 case clang::ARM::BI_InterlockedExchange16_nf:
246 case clang::ARM::BI_InterlockedExchange_nf:
247 case clang::ARM::BI_InterlockedExchange64_nf:
248 case clang::ARM::BI_InterlockedExchangePointer_nf:
249 return MSVCIntrin::_InterlockedExchange_nf;
250 case clang::ARM::BI_InterlockedCompareExchange8_acq:
251 case clang::ARM::BI_InterlockedCompareExchange16_acq:
252 case clang::ARM::BI_InterlockedCompareExchange_acq:
253 case clang::ARM::BI_InterlockedCompareExchange64_acq:
254 case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
255 return MSVCIntrin::_InterlockedCompareExchange_acq;
256 case clang::ARM::BI_InterlockedCompareExchange8_rel:
257 case clang::ARM::BI_InterlockedCompareExchange16_rel:
258 case clang::ARM::BI_InterlockedCompareExchange_rel:
259 case clang::ARM::BI_InterlockedCompareExchange64_rel:
260 case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
261 return MSVCIntrin::_InterlockedCompareExchange_rel;
262 case clang::ARM::BI_InterlockedCompareExchange8_nf:
263 case clang::ARM::BI_InterlockedCompareExchange16_nf:
264 case clang::ARM::BI_InterlockedCompareExchange_nf:
265 case clang::ARM::BI_InterlockedCompareExchange64_nf:
266 return MSVCIntrin::_InterlockedCompareExchange_nf;
267 case clang::ARM::BI_InterlockedOr8_acq:
268 case clang::ARM::BI_InterlockedOr16_acq:
269 case clang::ARM::BI_InterlockedOr_acq:
270 case clang::ARM::BI_InterlockedOr64_acq:
271 return MSVCIntrin::_InterlockedOr_acq;
272 case clang::ARM::BI_InterlockedOr8_rel:
273 case clang::ARM::BI_InterlockedOr16_rel:
274 case clang::ARM::BI_InterlockedOr_rel:
275 case clang::ARM::BI_InterlockedOr64_rel:
276 return MSVCIntrin::_InterlockedOr_rel;
277 case clang::ARM::BI_InterlockedOr8_nf:
278 case clang::ARM::BI_InterlockedOr16_nf:
279 case clang::ARM::BI_InterlockedOr_nf:
280 case clang::ARM::BI_InterlockedOr64_nf:
281 return MSVCIntrin::_InterlockedOr_nf;
282 case clang::ARM::BI_InterlockedXor8_acq:
283 case clang::ARM::BI_InterlockedXor16_acq:
284 case clang::ARM::BI_InterlockedXor_acq:
285 case clang::ARM::BI_InterlockedXor64_acq:
286 return MSVCIntrin::_InterlockedXor_acq;
287 case clang::ARM::BI_InterlockedXor8_rel:
288 case clang::ARM::BI_InterlockedXor16_rel:
289 case clang::ARM::BI_InterlockedXor_rel:
290 case clang::ARM::BI_InterlockedXor64_rel:
291 return MSVCIntrin::_InterlockedXor_rel;
292 case clang::ARM::BI_InterlockedXor8_nf:
293 case clang::ARM::BI_InterlockedXor16_nf:
294 case clang::ARM::BI_InterlockedXor_nf:
295 case clang::ARM::BI_InterlockedXor64_nf:
296 return MSVCIntrin::_InterlockedXor_nf;
297 case clang::ARM::BI_InterlockedAnd8_acq:
298 case clang::ARM::BI_InterlockedAnd16_acq:
299 case clang::ARM::BI_InterlockedAnd_acq:
300 case clang::ARM::BI_InterlockedAnd64_acq:
301 return MSVCIntrin::_InterlockedAnd_acq;
302 case clang::ARM::BI_InterlockedAnd8_rel:
303 case clang::ARM::BI_InterlockedAnd16_rel:
304 case clang::ARM::BI_InterlockedAnd_rel:
305 case clang::ARM::BI_InterlockedAnd64_rel:
306 return MSVCIntrin::_InterlockedAnd_rel;
307 case clang::ARM::BI_InterlockedAnd8_nf:
308 case clang::ARM::BI_InterlockedAnd16_nf:
309 case clang::ARM::BI_InterlockedAnd_nf:
310 case clang::ARM::BI_InterlockedAnd64_nf:
311 return MSVCIntrin::_InterlockedAnd_nf;
312 case clang::ARM::BI_InterlockedIncrement16_acq:
313 case clang::ARM::BI_InterlockedIncrement_acq:
314 case clang::ARM::BI_InterlockedIncrement64_acq:
315 return MSVCIntrin::_InterlockedIncrement_acq;
316 case clang::ARM::BI_InterlockedIncrement16_rel:
317 case clang::ARM::BI_InterlockedIncrement_rel:
318 case clang::ARM::BI_InterlockedIncrement64_rel:
319 return MSVCIntrin::_InterlockedIncrement_rel;
320 case clang::ARM::BI_InterlockedIncrement16_nf:
321 case clang::ARM::BI_InterlockedIncrement_nf:
322 case clang::ARM::BI_InterlockedIncrement64_nf:
323 return MSVCIntrin::_InterlockedIncrement_nf;
324 case clang::ARM::BI_InterlockedDecrement16_acq:
325 case clang::ARM::BI_InterlockedDecrement_acq:
326 case clang::ARM::BI_InterlockedDecrement64_acq:
327 return MSVCIntrin::_InterlockedDecrement_acq;
328 case clang::ARM::BI_InterlockedDecrement16_rel:
329 case clang::ARM::BI_InterlockedDecrement_rel:
330 case clang::ARM::BI_InterlockedDecrement64_rel:
331 return MSVCIntrin::_InterlockedDecrement_rel;
332 case clang::ARM::BI_InterlockedDecrement16_nf:
333 case clang::ARM::BI_InterlockedDecrement_nf:
334 case clang::ARM::BI_InterlockedDecrement64_nf:
335 return MSVCIntrin::_InterlockedDecrement_nf;
336 }
337 llvm_unreachable("must return from switch");
338}
339
340
341
343 unsigned IntrinsicID,
344 unsigned ConstrainedIntrinsicID,
345 llvm::Type *Ty,
347 Function *F;
348 if (CGF.Builder.getIsFPConstrained())
350 else
352
353 if (CGF.Builder.getIsFPConstrained())
354 return CGF.Builder.CreateConstrainedFPCall(F, Args);
355 else
356 return CGF.Builder.CreateCall(F, Args);
357}
358
361 bool HasFastHalfType = true,
362 bool V1Ty = false,
363 bool AllowBFloatArgsAndRet = true) {
364 int IsQuad = TypeFlags.isQuad();
369 return llvm::FixedVectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
372 return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
374 if (AllowBFloatArgsAndRet)
375 return llvm::FixedVectorType::get(CGF->BFloatTy, V1Ty ? 1 : (4 << IsQuad));
376 else
377 return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
379 if (HasFastHalfType)
380 return llvm::FixedVectorType::get(CGF->HalfTy, V1Ty ? 1 : (4 << IsQuad));
381 else
382 return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
384 return llvm::FixedVectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad));
387 return llvm::FixedVectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad));
389
390
391
392 return llvm::FixedVectorType::get(CGF->Int8Ty, 16);
394 return llvm::FixedVectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad));
396 return llvm::FixedVectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad));
397 }
398 llvm_unreachable("Unknown vector element type!");
399}
400
403 int IsQuad = IntTypeFlags.isQuad();
406 return llvm::FixedVectorType::get(CGF->HalfTy, (4 << IsQuad));
408 return llvm::FixedVectorType::get(CGF->FloatTy, (2 << IsQuad));
410 return llvm::FixedVectorType::get(CGF->DoubleTy, (1 << IsQuad));
411 default:
412 llvm_unreachable("Type can't be converted to floating-point!");
413 }
414}
415
417 const ElementCount &Count) {
418 Value *SV = llvm::ConstantVector::getSplat(Count, C);
419 return Builder.CreateShuffleVector(V, V, SV, "lane");
420}
421
425}
426
428 const char *name,
429 unsigned shift, bool rightshift) {
430 unsigned j = 0;
431 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
432 ai != ae; ++ai, ++j) {
433 if (F->isConstrainedFPIntrinsic())
434 if (ai->getType()->isMetadataTy())
435 continue;
436 if (shift > 0 && shift == j)
438 else
439 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
440 }
441
442 if (F->isConstrainedFPIntrinsic())
443 return Builder.CreateConstrainedFPCall(F, Ops, name);
444 else
445 return Builder.CreateCall(F, Ops, name);
446}
447
451 const CallExpr *E, const char *name) {
452 llvm::Value *FPM =
454 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_set_fpmr), FPM);
455 return EmitNeonCall(CGM.getIntrinsic(IID, Tys), Ops, name);
456}
457
459 unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
461
462 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
463 RetTy->getPrimitiveSizeInBits();
464 llvm::Type *Tys[] = {llvm::FixedVectorType::get(RetTy, ElemCount),
465 Ops[1]->getType()};
466 if (ExtendLaneArg) {
467 auto *VT = llvm::FixedVectorType::get(Int8Ty, 16);
468 Ops[2] = Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
469 uint64_t(0));
470 }
472}
473
475 unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy,
477
478 if (ExtendLaneArg) {
479 auto *VT = llvm::FixedVectorType::get(Int8Ty, 16);
480 Ops[2] = Builder.CreateInsertVector(VT, PoisonValue::get(VT), Ops[2],
481 uint64_t(0));
482 }
483 const unsigned ElemCount = Ops[0]->getType()->getPrimitiveSizeInBits() /
484 RetTy->getPrimitiveSizeInBits();
485 return EmitFP8NeonCall(IID, {llvm::FixedVectorType::get(RetTy, ElemCount)},
486 Ops, E, name);
487}
488
490 bool neg) {
492 return ConstantInt::getSigned(Ty, neg ? -SV : SV);
493}
494
496 llvm::Type *Ty1, bool Extract,
499 const char *name) {
500 llvm::Type *Tys[] = {Ty0, Ty1};
501 if (Extract) {
502
503
504 Tys[1] = llvm::FixedVectorType::get(Int8Ty, 8);
505 Ops[0] = Builder.CreateExtractVector(Tys[1], Ops[0], uint64_t(0));
506 }
508}
509
510
512 llvm::Type *Ty, bool usgn,
513 const char *name) {
515
517 int EltSize = VTy->getScalarSizeInBits();
518
519 Vec = Builder.CreateBitCast(Vec, Ty);
520
521
522
523 if (ShiftAmt == EltSize) {
524 if (usgn) {
525
526 return llvm::ConstantAggregateZero::get(VTy);
527 } else {
528
529
530 --ShiftAmt;
531 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
532 }
533 }
534
536 if (usgn)
537 return Builder.CreateLShr(Vec, Shift, name);
538 else
539 return Builder.CreateAShr(Vec, Shift, name);
540}
541
542enum {
546
549
552
555
562};
563
564namespace {
565struct ARMVectorIntrinsicInfo {
566 const char *NameHint;
567 unsigned BuiltinID;
568 unsigned LLVMIntrinsic;
569 unsigned AltLLVMIntrinsic;
571
572 bool operator<(unsigned RHSBuiltinID) const {
573 return BuiltinID < RHSBuiltinID;
574 }
575 bool operator<(const ARMVectorIntrinsicInfo &TE) const {
576 return BuiltinID < TE.BuiltinID;
577 }
578};
579}
580
581#define NEONMAP0(NameBase) \
582 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
583
584#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
585 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
586 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
587
588#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
589 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
590 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
591 TypeModifier }
592
593
595 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
602 NEONMAP1(vabs_v, arm_neon_vabs, 0),
603 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
607 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
608 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
609 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
610 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
611 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
612 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
613 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
614 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
615 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
628 NEONMAP1(vcage_v, arm_neon_vacge, 0),
629 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
630 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
631 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
632 NEONMAP1(vcale_v, arm_neon_vacge, 0),
633 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
634 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
635 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
652 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
655 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
657 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
658 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
659 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
660 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
661 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
662 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
663 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
664 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
665 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
672 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
673 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
674 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
675 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
676 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
677 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
678 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
679 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
680 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
681 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
682 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
683 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
684 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
685 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
686 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
687 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
688 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
689 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
690 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
691 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
692 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
693 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
694 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
695 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
696 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
697 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
698 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
699 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
700 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
701 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
702 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
703 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
704 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
705 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
706 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
707 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
708 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
709 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
710 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
711 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
712 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
713 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
714 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
715 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
716 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
717 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
718 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
719 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
720 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
724 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
725 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
726 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
727 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
728 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
729 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
730 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
731 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
732 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
739 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
740 NEONMAP1(vdot_u32, arm_neon_udot, 0),
741 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
742 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
752 NEONMAP1(vld1_v, arm_neon_vld1, 0),
753 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
754 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
755 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
757 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
758 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
759 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
760 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
761 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
762 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
763 NEONMAP1(vld2_v, arm_neon_vld2, 0),
764 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
765 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
766 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
767 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
768 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
769 NEONMAP1(vld3_v, arm_neon_vld3, 0),
770 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
771 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
772 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
773 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
774 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
775 NEONMAP1(vld4_v, arm_neon_vld4, 0),
776 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
777 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
778 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
787 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
788 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
806 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
807 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
831 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
832 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
836 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
837 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
860 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
861 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
865 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
866 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
867 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
868 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
869 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
870 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
879 NEONMAP1(vst1_v, arm_neon_vst1, 0),
880 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
881 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
882 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
883 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
884 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
885 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
886 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
887 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
888 NEONMAP1(vst2_v, arm_neon_vst2, 0),
889 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
890 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
891 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
892 NEONMAP1(vst3_v, arm_neon_vst3, 0),
893 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
894 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
895 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
896 NEONMAP1(vst4_v, arm_neon_vst4, 0),
897 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
898 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
904 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
905 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
906 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
911};
912
918 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
919 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
924 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
925 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
926 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
927 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
936 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
937 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
938 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
939 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
940 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
951 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
952 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
953 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
954 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
955 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
956 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
957 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
958 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
995 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
998 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
1000 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
1001 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
1002 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
1003 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
1004 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
1005 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
1006 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
1007 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
1008 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
1009 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
1013 NEONMAP0(vcvtq_high_bf16_f32),
1014 NEONMAP0(vcvtq_low_bf16_f32),
1015 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
1016 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
1017 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
1018 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
1019 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
1020 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
1021 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
1022 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
1023 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
1024 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
1026 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
1027 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
1028 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
1029 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
1042 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
1043 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
1044 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
1045 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
1046 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
1047 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
1048 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
1049 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
1054 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
1055 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
1056 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
1057 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
1058 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
1059 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
1060 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
1061 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
1068 NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
1074 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
1075 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
1076 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
1077 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
1079 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
1080 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
1095 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
1096 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
1098 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
1099 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
1107 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
1108 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
1112 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
1113 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
1114 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
1141 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
1142 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
1146 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
1147 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
1148 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
1149 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
1150 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
1151 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
1152 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
1153 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
1154 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
1155 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
1164 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
1165 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
1166 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
1167 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
1168 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
1169 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
1170 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
1171 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
1172 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
1173 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
1174 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
1175 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
1176 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
1177 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
1178 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
1182 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
1183 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
1184 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
1185 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
1186};
1187
1270 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
1307 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
1335 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
1416 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
1417 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
1418 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
1419 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
1434
1469};
1470
1471
1472
1474 { NEON::BI__builtin_neon_splat_lane_bf16, NEON::BI__builtin_neon_splat_lane_v, },
1475 { NEON::BI__builtin_neon_splat_laneq_bf16, NEON::BI__builtin_neon_splat_laneq_v, },
1476 { NEON::BI__builtin_neon_splatq_lane_bf16, NEON::BI__builtin_neon_splatq_lane_v, },
1477 { NEON::BI__builtin_neon_splatq_laneq_bf16, NEON::BI__builtin_neon_splatq_laneq_v, },
1478 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
1479 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
1480 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
1481 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
1482 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
1483 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
1484 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
1485 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
1486 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
1487 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
1488 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
1489 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
1490 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
1491 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
1492 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
1493 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
1494 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
1495 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
1496 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
1497 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
1498 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
1499 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
1500 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
1501 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
1502 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
1503 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
1504 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
1505 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
1506 { NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v },
1507 { NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v },
1508 { NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v },
1509 { NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v },
1510 { NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v },
1511 { NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v },
1512 { NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v },
1513 { NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v },
1514 { NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v },
1515 { NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v },
1516 { NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v },
1517 { NEON::BI__builtin_neon_vld1q_lane_bf16, NEON::BI__builtin_neon_vld1q_lane_v },
1518 { NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v },
1519 { NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v },
1520 { NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v },
1521 { NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v },
1522 { NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v },
1523 { NEON::BI__builtin_neon_vld2q_lane_bf16, NEON::BI__builtin_neon_vld2q_lane_v },
1524 { NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v },
1525 { NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v },
1526 { NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v },
1527 { NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v },
1528 { NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v },
1529 { NEON::BI__builtin_neon_vld3q_lane_bf16, NEON::BI__builtin_neon_vld3q_lane_v },
1530 { NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v },
1531 { NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v },
1532 { NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v },
1533 { NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v },
1534 { NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v },
1535 { NEON::BI__builtin_neon_vld4q_lane_bf16, NEON::BI__builtin_neon_vld4q_lane_v },
1536 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
1537 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
1538 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
1539 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
1540 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
1541 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
1542 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
1543 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
1544 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
1545 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
1546 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
1547 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
1548 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
1549 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
1550 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
1551 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
1552 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
1553 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
1554 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
1555 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
1556 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
1557 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
1558 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
1559 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
1560 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
1561 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
1562 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
1563 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
1564 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
1565 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
1566 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
1567 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
1568 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
1569 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
1570 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
1571 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
1572 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
1573 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
1574 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
1575 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
1576 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
1577 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
1578 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
1579 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
1580 { NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v },
1581 { NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v },
1582 { NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v },
1583 { NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v },
1584 { NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v },
1585 { NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v },
1586 { NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v },
1587 { NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v },
1588 { NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v },
1589 { NEON::BI__builtin_neon_vst1q_lane_bf16, NEON::BI__builtin_neon_vst1q_lane_v },
1590 { NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v },
1591 { NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v },
1592 { NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v },
1593 { NEON::BI__builtin_neon_vst2q_lane_bf16, NEON::BI__builtin_neon_vst2q_lane_v },
1594 { NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v },
1595 { NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v },
1596 { NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v },
1597 { NEON::BI__builtin_neon_vst3q_lane_bf16, NEON::BI__builtin_neon_vst3q_lane_v },
1598 { NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v },
1599 { NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v },
1600 { NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v },
1601 { NEON::BI__builtin_neon_vst4q_lane_bf16, NEON::BI__builtin_neon_vst4q_lane_v },
1602
1603
1604
1605 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
1606 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
1607 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
1608 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
1609 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
1610 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
1611 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
1612 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
1613 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
1614 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
1615 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
1616 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
1617};
1618
1619#undef NEONMAP0
1620#undef NEONMAP1
1621#undef NEONMAP2
1622
1623#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
1624 { \
1625 #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
1626 TypeModifier \
1627 }
1628
1629#define SVEMAP2(NameBase, TypeModifier) \
1630 { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
1632#define GET_SVE_LLVM_INTRINSIC_MAP
1633#include "clang/Basic/arm_sve_builtin_cg.inc"
1634#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
1635#undef GET_SVE_LLVM_INTRINSIC_MAP
1636};
1637
1638#undef SVEMAP1
1639#undef SVEMAP2
1640
1641#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
1642 { \
1643 #NameBase, SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
1644 TypeModifier \
1645 }
1646
1647#define SMEMAP2(NameBase, TypeModifier) \
1648 { #NameBase, SME::BI__builtin_sme_##NameBase, 0, 0, TypeModifier }
1650#define GET_SME_LLVM_INTRINSIC_MAP
1651#include "clang/Basic/arm_sme_builtin_cg.inc"
1652#undef GET_SME_LLVM_INTRINSIC_MAP
1653};
1654
1655#undef SMEMAP1
1656#undef SMEMAP2
1657
1659
1664
1665static const ARMVectorIntrinsicInfo *
1667 unsigned BuiltinID, bool &MapProvenSorted) {
1668
1669#ifndef NDEBUG
1670 if (!MapProvenSorted) {
1671 assert(llvm::is_sorted(IntrinsicMap));
1672 MapProvenSorted = true;
1673 }
1674#endif
1675
1676 const ARMVectorIntrinsicInfo *Builtin =
1677 llvm::lower_bound(IntrinsicMap, BuiltinID);
1678
1679 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
1681
1682 return nullptr;
1683}
1684
1686 unsigned Modifier,
1689 int VectorSize = 0;
1691 VectorSize = 64;
1693 VectorSize = 128;
1694
1695
1700 Ty = llvm::FixedVectorType::get(
1701 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
1702
1703 Tys.push_back(Ty);
1704 }
1705
1706
1708 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
1709 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
1710 }
1711
1714
1717
1720
1721 return CGM.getIntrinsic(IntrinsicID, Tys);
1722}
1723
1725 CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo,
1727 unsigned BuiltinID = SISDInfo.BuiltinID;
1728 unsigned int Int = SISDInfo.LLVMIntrinsic;
1729 unsigned Modifier = SISDInfo.TypeModifier;
1730 const char *s = SISDInfo.NameHint;
1731
1732 switch (BuiltinID) {
1733 case NEON::BI__builtin_neon_vcled_s64:
1734 case NEON::BI__builtin_neon_vcled_u64:
1735 case NEON::BI__builtin_neon_vcles_f32:
1736 case NEON::BI__builtin_neon_vcled_f64:
1737 case NEON::BI__builtin_neon_vcltd_s64:
1738 case NEON::BI__builtin_neon_vcltd_u64:
1739 case NEON::BI__builtin_neon_vclts_f32:
1740 case NEON::BI__builtin_neon_vcltd_f64:
1741 case NEON::BI__builtin_neon_vcales_f32:
1742 case NEON::BI__builtin_neon_vcaled_f64:
1743 case NEON::BI__builtin_neon_vcalts_f32:
1744 case NEON::BI__builtin_neon_vcaltd_f64:
1745
1746
1747
1748 std::swap(Ops[0], Ops[1]);
1749 break;
1750 }
1751
1752 assert(Int && "Generic code assumes a valid intrinsic");
1753
1754
1758
1759 int j = 0;
1760 ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0);
1761 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
1762 ai != ae; ++ai, ++j) {
1763 llvm::Type *ArgTy = ai->getType();
1764 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
1765 ArgTy->getPrimitiveSizeInBits())
1766 continue;
1767
1768 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
1769
1770
1771 Ops[j] = CGF.Builder.CreateTruncOrBitCast(
1773 Ops[j] =
1774 CGF.Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
1775 }
1776
1779 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
1780 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
1781 return CGF.Builder.CreateExtractElement(Result, C0);
1782
1783 return CGF.Builder.CreateBitCast(Result, ResultType, s);
1784}
1785
1787 unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
1788 const char *NameHint, unsigned Modifier, const CallExpr *E,
1790 llvm::Triple::ArchType Arch) {
1791
1793 std::optionalllvm::APSInt NeonTypeConst =
1795 if (!NeonTypeConst)
1796 return nullptr;
1797
1798
1800 const bool Usgn = Type.isUnsigned();
1801 const bool Quad = Type.isQuad();
1802 const bool Floating = Type.isFloatingPoint();
1804 const bool AllowBFloatArgsAndRet =
1806
1807 llvm::FixedVectorType *VTy =
1808 GetNeonType(this, Type, HasFastHalfType, false, AllowBFloatArgsAndRet);
1809 llvm::Type *Ty = VTy;
1810 if (!Ty)
1811 return nullptr;
1812
1813 auto getAlignmentValue32 = [&](Address addr) -> Value* {
1814 return Builder.getInt32(addr.getAlignment().getQuantity());
1815 };
1816
1817 unsigned Int = LLVMIntrinsic;
1819 Int = AltLLVMIntrinsic;
1820
1821 switch (BuiltinID) {
1822 default: break;
1823 case NEON::BI__builtin_neon_splat_lane_v:
1824 case NEON::BI__builtin_neon_splat_laneq_v:
1825 case NEON::BI__builtin_neon_splatq_lane_v:
1826 case NEON::BI__builtin_neon_splatq_laneq_v: {
1827 auto NumElements = VTy->getElementCount();
1828 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
1829 NumElements = NumElements * 2;
1830 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
1831 NumElements = NumElements.divideCoefficientBy(2);
1832
1833 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
1835 }
1836 case NEON::BI__builtin_neon_vpadd_v:
1837 case NEON::BI__builtin_neon_vpaddq_v:
1838
1839 if (VTy->getElementType()->isFloatingPointTy() &&
1840 Int == Intrinsic::aarch64_neon_addp)
1841 Int = Intrinsic::aarch64_neon_faddp;
1842 break;
1843 case NEON::BI__builtin_neon_vabs_v:
1844 case NEON::BI__builtin_neon_vabsq_v:
1845 if (VTy->getElementType()->isFloatingPointTy())
1846 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs");
1847 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs");
1848 case NEON::BI__builtin_neon_vadd_v:
1849 case NEON::BI__builtin_neon_vaddq_v: {
1850 llvm::Type *VTy = llvm::FixedVectorType::get(Int8Ty, Quad ? 16 : 8);
1851 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
1852 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
1853 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
1854 return Builder.CreateBitCast(Ops[0], Ty);
1855 }
1856 case NEON::BI__builtin_neon_vaddhn_v: {
1857 llvm::FixedVectorType *SrcTy =
1858 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
1859
1860
1861 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
1862 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
1863 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
1864
1865
1866 Constant *ShiftAmt =
1867 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
1868 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
1869
1870
1871 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn");
1872 }
1873 case NEON::BI__builtin_neon_vcale_v:
1874 case NEON::BI__builtin_neon_vcaleq_v:
1875 case NEON::BI__builtin_neon_vcalt_v:
1876 case NEON::BI__builtin_neon_vcaltq_v:
1877 std::swap(Ops[0], Ops[1]);
1878 [[fallthrough]];
1879 case NEON::BI__builtin_neon_vcage_v:
1880 case NEON::BI__builtin_neon_vcageq_v:
1881 case NEON::BI__builtin_neon_vcagt_v:
1882 case NEON::BI__builtin_neon_vcagtq_v: {
1883 llvm::Type *Ty;
1884 switch (VTy->getScalarSizeInBits()) {
1885 default: llvm_unreachable("unexpected type");
1886 case 32:
1888 break;
1889 case 64:
1891 break;
1892 case 16:
1894 break;
1895 }
1896 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
1897 llvm::Type *Tys[] = { VTy, VecFlt };
1898 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
1900 }
1901 case NEON::BI__builtin_neon_vceqz_v:
1902 case NEON::BI__builtin_neon_vceqzq_v:
1904 Ops[0], Ty, Floating ? ICmpInst::FCMP_OEQ : ICmpInst::ICMP_EQ, "vceqz");
1905 case NEON::BI__builtin_neon_vcgez_v:
1906 case NEON::BI__builtin_neon_vcgezq_v:
1908 Ops[0], Ty, Floating ? ICmpInst::FCMP_OGE : ICmpInst::ICMP_SGE,
1909 "vcgez");
1910 case NEON::BI__builtin_neon_vclez_v:
1911 case NEON::BI__builtin_neon_vclezq_v:
1913 Ops[0], Ty, Floating ? ICmpInst::FCMP_OLE : ICmpInst::ICMP_SLE,
1914 "vclez");
1915 case NEON::BI__builtin_neon_vcgtz_v:
1916 case NEON::BI__builtin_neon_vcgtzq_v:
1918 Ops[0], Ty, Floating ? ICmpInst::FCMP_OGT : ICmpInst::ICMP_SGT,
1919 "vcgtz");
1920 case NEON::BI__builtin_neon_vcltz_v:
1921 case NEON::BI__builtin_neon_vcltzq_v:
1923 Ops[0], Ty, Floating ? ICmpInst::FCMP_OLT : ICmpInst::ICMP_SLT,
1924 "vcltz");
1925 case NEON::BI__builtin_neon_vclz_v:
1926 case NEON::BI__builtin_neon_vclzq_v:
1927
1928
1929 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
1930 break;
1931 case NEON::BI__builtin_neon_vcvt_f32_v:
1932 case NEON::BI__builtin_neon_vcvtq_f32_v:
1933 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1935 HasFastHalfType);
1936 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
1937 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
1938 case NEON::BI__builtin_neon_vcvt_f16_s16:
1939 case NEON::BI__builtin_neon_vcvt_f16_u16:
1940 case NEON::BI__builtin_neon_vcvtq_f16_s16:
1941 case NEON::BI__builtin_neon_vcvtq_f16_u16:
1942 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1944 HasFastHalfType);
1945 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
1946 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
1947 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
1948 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
1949 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
1950 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
1952 Function *F = CGM.getIntrinsic(Int, Tys);
1954 }
1955 case NEON::BI__builtin_neon_vcvt_n_f32_v:
1956 case NEON::BI__builtin_neon_vcvt_n_f64_v:
1957 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
1958 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
1960 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
1961 Function *F = CGM.getIntrinsic(Int, Tys);
1963 }
1964 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
1965 case NEON::BI__builtin_neon_vcvt_n_s32_v:
1966 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
1967 case NEON::BI__builtin_neon_vcvt_n_u32_v:
1968 case NEON::BI__builtin_neon_vcvt_n_s64_v:
1969 case NEON::BI__builtin_neon_vcvt_n_u64_v:
1970 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
1971 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
1972 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
1973 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
1974 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
1975 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
1977 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
1979 }
1980 case NEON::BI__builtin_neon_vcvt_s32_v:
1981 case NEON::BI__builtin_neon_vcvt_u32_v:
1982 case NEON::BI__builtin_neon_vcvt_s64_v:
1983 case NEON::BI__builtin_neon_vcvt_u64_v:
1984 case NEON::BI__builtin_neon_vcvt_s16_f16:
1985 case NEON::BI__builtin_neon_vcvt_u16_f16:
1986 case NEON::BI__builtin_neon_vcvtq_s32_v:
1987 case NEON::BI__builtin_neon_vcvtq_u32_v:
1988 case NEON::BI__builtin_neon_vcvtq_s64_v:
1989 case NEON::BI__builtin_neon_vcvtq_u64_v:
1990 case NEON::BI__builtin_neon_vcvtq_s16_f16:
1991 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
1993 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
1994 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
1995 }
1996 case NEON::BI__builtin_neon_vcvta_s16_f16:
1997 case NEON::BI__builtin_neon_vcvta_s32_v:
1998 case NEON::BI__builtin_neon_vcvta_s64_v:
1999 case NEON::BI__builtin_neon_vcvta_u16_f16:
2000 case NEON::BI__builtin_neon_vcvta_u32_v:
2001 case NEON::BI__builtin_neon_vcvta_u64_v:
2002 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
2003 case NEON::BI__builtin_neon_vcvtaq_s32_v:
2004 case NEON::BI__builtin_neon_vcvtaq_s64_v:
2005 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
2006 case NEON::BI__builtin_neon_vcvtaq_u32_v:
2007 case NEON::BI__builtin_neon_vcvtaq_u64_v:
2008 case NEON::BI__builtin_neon_vcvtn_s16_f16:
2009 case NEON::BI__builtin_neon_vcvtn_s32_v:
2010 case NEON::BI__builtin_neon_vcvtn_s64_v:
2011 case NEON::BI__builtin_neon_vcvtn_u16_f16:
2012 case NEON::BI__builtin_neon_vcvtn_u32_v:
2013 case NEON::BI__builtin_neon_vcvtn_u64_v:
2014 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
2015 case NEON::BI__builtin_neon_vcvtnq_s32_v:
2016 case NEON::BI__builtin_neon_vcvtnq_s64_v:
2017 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
2018 case NEON::BI__builtin_neon_vcvtnq_u32_v:
2019 case NEON::BI__builtin_neon_vcvtnq_u64_v:
2020 case NEON::BI__builtin_neon_vcvtp_s16_f16:
2021 case NEON::BI__builtin_neon_vcvtp_s32_v:
2022 case NEON::BI__builtin_neon_vcvtp_s64_v:
2023 case NEON::BI__builtin_neon_vcvtp_u16_f16:
2024 case NEON::BI__builtin_neon_vcvtp_u32_v:
2025 case NEON::BI__builtin_neon_vcvtp_u64_v:
2026 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
2027 case NEON::BI__builtin_neon_vcvtpq_s32_v:
2028 case NEON::BI__builtin_neon_vcvtpq_s64_v:
2029 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
2030 case NEON::BI__builtin_neon_vcvtpq_u32_v:
2031 case NEON::BI__builtin_neon_vcvtpq_u64_v:
2032 case NEON::BI__builtin_neon_vcvtm_s16_f16:
2033 case NEON::BI__builtin_neon_vcvtm_s32_v:
2034 case NEON::BI__builtin_neon_vcvtm_s64_v:
2035 case NEON::BI__builtin_neon_vcvtm_u16_f16:
2036 case NEON::BI__builtin_neon_vcvtm_u32_v:
2037 case NEON::BI__builtin_neon_vcvtm_u64_v:
2038 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
2039 case NEON::BI__builtin_neon_vcvtmq_s32_v:
2040 case NEON::BI__builtin_neon_vcvtmq_s64_v:
2041 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
2042 case NEON::BI__builtin_neon_vcvtmq_u32_v:
2043 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
2045 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
2046 }
2047 case NEON::BI__builtin_neon_vcvtx_f32_v: {
2048 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
2049 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
2050
2051 }
2052 case NEON::BI__builtin_neon_vext_v:
2053 case NEON::BI__builtin_neon_vextq_v: {
2056 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
2057 Indices.push_back(i+CV);
2058
2059 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2060 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2061 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext");
2062 }
2063 case NEON::BI__builtin_neon_vfma_v:
2064 case NEON::BI__builtin_neon_vfmaq_v: {
2065 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2066 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2067 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2068
2069
2071 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
2072 {Ops[1], Ops[2], Ops[0]});
2073 }
2074 case NEON::BI__builtin_neon_vld1_v:
2075 case NEON::BI__builtin_neon_vld1q_v: {
2076 llvm::Type *Tys[] = {Ty, Int8PtrTy};
2077 Ops.push_back(getAlignmentValue32(PtrOp0));
2078 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1");
2079 }
2080 case NEON::BI__builtin_neon_vld1_x2_v:
2081 case NEON::BI__builtin_neon_vld1q_x2_v:
2082 case NEON::BI__builtin_neon_vld1_x3_v:
2083 case NEON::BI__builtin_neon_vld1q_x3_v:
2084 case NEON::BI__builtin_neon_vld1_x4_v:
2085 case NEON::BI__builtin_neon_vld1q_x4_v: {
2087 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
2088 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN");
2089 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
2090 }
2091 case NEON::BI__builtin_neon_vld2_v:
2092 case NEON::BI__builtin_neon_vld2q_v:
2093 case NEON::BI__builtin_neon_vld3_v:
2094 case NEON::BI__builtin_neon_vld3q_v:
2095 case NEON::BI__builtin_neon_vld4_v:
2096 case NEON::BI__builtin_neon_vld4q_v:
2097 case NEON::BI__builtin_neon_vld2_dup_v:
2098 case NEON::BI__builtin_neon_vld2q_dup_v:
2099 case NEON::BI__builtin_neon_vld3_dup_v:
2100 case NEON::BI__builtin_neon_vld3q_dup_v:
2101 case NEON::BI__builtin_neon_vld4_dup_v:
2102 case NEON::BI__builtin_neon_vld4q_dup_v: {
2103 llvm::Type *Tys[] = {Ty, Int8PtrTy};
2104 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
2105 Value *Align = getAlignmentValue32(PtrOp1);
2106 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
2107 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
2108 }
2109 case NEON::BI__builtin_neon_vld1_dup_v:
2110 case NEON::BI__builtin_neon_vld1q_dup_v: {
2111 Value *V = PoisonValue::get(Ty);
2113 LoadInst *Ld = Builder.CreateLoad(PtrOp0);
2114 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
2115 Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
2117 }
2118 case NEON::BI__builtin_neon_vld2_lane_v:
2119 case NEON::BI__builtin_neon_vld2q_lane_v:
2120 case NEON::BI__builtin_neon_vld3_lane_v:
2121 case NEON::BI__builtin_neon_vld3q_lane_v:
2122 case NEON::BI__builtin_neon_vld4_lane_v:
2123 case NEON::BI__builtin_neon_vld4q_lane_v: {
2124 llvm::Type *Tys[] = {Ty, Int8PtrTy};
2125 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
2126 for (unsigned I = 2; I < Ops.size() - 1; ++I)
2127 Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
2128 Ops.push_back(getAlignmentValue32(PtrOp1));
2129 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), NameHint);
2130 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
2131 }
2132 case NEON::BI__builtin_neon_vmovl_v: {
2133 llvm::FixedVectorType *DTy =
2134 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
2135 Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
2136 if (Usgn)
2137 return Builder.CreateZExt(Ops[0], Ty, "vmovl");
2138 return Builder.CreateSExt(Ops[0], Ty, "vmovl");
2139 }
2140 case NEON::BI__builtin_neon_vmovn_v: {
2141 llvm::FixedVectorType *QTy =
2142 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
2143 Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
2144 return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
2145 }
2146 case NEON::BI__builtin_neon_vmull_v:
2147
2148
2149
2150
2151
2152 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
2153 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
2154 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
2155 case NEON::BI__builtin_neon_vpadal_v:
2156 case NEON::BI__builtin_neon_vpadalq_v: {
2157
2158 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
2159 llvm::Type *EltTy =
2160 llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
2161 auto *NarrowTy =
2162 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
2163 llvm::Type *Tys[2] = { Ty, NarrowTy };
2164 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
2165 }
2166 case NEON::BI__builtin_neon_vpaddl_v:
2167 case NEON::BI__builtin_neon_vpaddlq_v: {
2168
2169 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
2170 llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
2171 auto *NarrowTy =
2172 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
2173 llvm::Type *Tys[2] = { Ty, NarrowTy };
2174 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl");
2175 }
2176 case NEON::BI__builtin_neon_vqdmlal_v:
2177 case NEON::BI__builtin_neon_vqdmlsl_v: {
2179 Ops[1] =
2180 EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal");
2181 Ops.resize(2);
2182 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
2183 }
2184 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
2185 case NEON::BI__builtin_neon_vqdmulh_lane_v:
2186 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
2187 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
2189 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
2190 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
2191 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
2192 RTy->getNumElements() * 2);
2193 llvm::Type *Tys[2] = {
2195 false))};
2196 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
2197 }
2198 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
2199 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
2200 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
2201 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
2202 llvm::Type *Tys[2] = {
2204 true))};
2205 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
2206 }
2207 case NEON::BI__builtin_neon_vqshl_n_v:
2208 case NEON::BI__builtin_neon_vqshlq_n_v:
2209 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n",
2210 1, false);
2211 case NEON::BI__builtin_neon_vqshlu_n_v:
2212 case NEON::BI__builtin_neon_vqshluq_n_v:
2213 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n",
2214 1, false);
2215 case NEON::BI__builtin_neon_vrecpe_v:
2216 case NEON::BI__builtin_neon_vrecpeq_v:
2217 case NEON::BI__builtin_neon_vrsqrte_v:
2218 case NEON::BI__builtin_neon_vrsqrteq_v:
2219 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
2220 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
2221 case NEON::BI__builtin_neon_vrndi_v:
2222 case NEON::BI__builtin_neon_vrndiq_v:
2223 Int = Builder.getIsFPConstrained()
2224 ? Intrinsic::experimental_constrained_nearbyint
2225 : Intrinsic::nearbyint;
2226 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
2227 case NEON::BI__builtin_neon_vrshr_n_v:
2228 case NEON::BI__builtin_neon_vrshrq_n_v:
2229 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n",
2230 1, true);
2231 case NEON::BI__builtin_neon_vsha512hq_u64:
2232 case NEON::BI__builtin_neon_vsha512h2q_u64:
2233 case NEON::BI__builtin_neon_vsha512su0q_u64:
2234 case NEON::BI__builtin_neon_vsha512su1q_u64: {
2237 }
2238 case NEON::BI__builtin_neon_vshl_n_v:
2239 case NEON::BI__builtin_neon_vshlq_n_v:
2241 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
2242 "vshl_n");
2243 case NEON::BI__builtin_neon_vshll_n_v: {
2244 llvm::FixedVectorType *SrcTy =
2245 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
2246 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
2247 if (Usgn)
2248 Ops[0] = Builder.CreateZExt(Ops[0], VTy);
2249 else
2250 Ops[0] = Builder.CreateSExt(Ops[0], VTy);
2252 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n");
2253 }
2254 case NEON::BI__builtin_neon_vshrn_n_v: {
2255 llvm::FixedVectorType *SrcTy =
2256 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
2257 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
2259 if (Usgn)
2260 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
2261 else
2262 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
2263 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n");
2264 }
2265 case NEON::BI__builtin_neon_vshr_n_v:
2266 case NEON::BI__builtin_neon_vshrq_n_v:
2268 case NEON::BI__builtin_neon_vst1_v:
2269 case NEON::BI__builtin_neon_vst1q_v:
2270 case NEON::BI__builtin_neon_vst2_v:
2271 case NEON::BI__builtin_neon_vst2q_v:
2272 case NEON::BI__builtin_neon_vst3_v:
2273 case NEON::BI__builtin_neon_vst3q_v:
2274 case NEON::BI__builtin_neon_vst4_v:
2275 case NEON::BI__builtin_neon_vst4q_v:
2276 case NEON::BI__builtin_neon_vst2_lane_v:
2277 case NEON::BI__builtin_neon_vst2q_lane_v:
2278 case NEON::BI__builtin_neon_vst3_lane_v:
2279 case NEON::BI__builtin_neon_vst3q_lane_v:
2280 case NEON::BI__builtin_neon_vst4_lane_v:
2281 case NEON::BI__builtin_neon_vst4q_lane_v: {
2282 llvm::Type *Tys[] = {Int8PtrTy, Ty};
2283 Ops.push_back(getAlignmentValue32(PtrOp0));
2284 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "");
2285 }
2286 case NEON::BI__builtin_neon_vsm3partw1q_u32:
2287 case NEON::BI__builtin_neon_vsm3partw2q_u32:
2288 case NEON::BI__builtin_neon_vsm3ss1q_u32:
2289 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
2290 case NEON::BI__builtin_neon_vsm4eq_u32: {
2293 }
2294 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
2295 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
2296 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
2297 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
2301 }
2302 case NEON::BI__builtin_neon_vst1_x2_v:
2303 case NEON::BI__builtin_neon_vst1q_x2_v:
2304 case NEON::BI__builtin_neon_vst1_x3_v:
2305 case NEON::BI__builtin_neon_vst1q_x3_v:
2306 case NEON::BI__builtin_neon_vst1_x4_v:
2307 case NEON::BI__builtin_neon_vst1q_x4_v: {
2308
2309
2310 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
2311 Arch == llvm::Triple::aarch64_32) {
2313 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
2314 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "");
2315 }
2317 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "");
2318 }
2319 case NEON::BI__builtin_neon_vsubhn_v: {
2320 llvm::FixedVectorType *SrcTy =
2321 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
2322
2323
2324 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
2325 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
2326 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
2327
2328
2329 Constant *ShiftAmt =
2330 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
2331 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
2332
2333
2334 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn");
2335 }
2336 case NEON::BI__builtin_neon_vtrn_v:
2337 case NEON::BI__builtin_neon_vtrnq_v: {
2338 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2339 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2340 Value *SV = nullptr;
2341
2342 for (unsigned vi = 0; vi != 2; ++vi) {
2344 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
2345 Indices.push_back(i+vi);
2346 Indices.push_back(i+e+vi);
2347 }
2348 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
2349 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
2350 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
2351 }
2352 return SV;
2353 }
2354 case NEON::BI__builtin_neon_vtst_v:
2355 case NEON::BI__builtin_neon_vtstq_v: {
2356 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2357 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2358 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
2359 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
2360 ConstantAggregateZero::get(Ty));
2361 return Builder.CreateSExt(Ops[0], Ty, "vtst");
2362 }
2363 case NEON::BI__builtin_neon_vuzp_v:
2364 case NEON::BI__builtin_neon_vuzpq_v: {
2365 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2366 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2367 Value *SV = nullptr;
2368
2369 for (unsigned vi = 0; vi != 2; ++vi) {
2371 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
2372 Indices.push_back(2*i+vi);
2373
2374 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
2375 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
2376 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
2377 }
2378 return SV;
2379 }
2380 case NEON::BI__builtin_neon_vxarq_u64: {
2384 }
2385 case NEON::BI__builtin_neon_vzip_v:
2386 case NEON::BI__builtin_neon_vzipq_v: {
2387 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2388 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2389 Value *SV = nullptr;
2390
2391 for (unsigned vi = 0; vi != 2; ++vi) {
2393 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
2394 Indices.push_back((i + vi*e) >> 1);
2395 Indices.push_back(((i + vi*e) >> 1)+e);
2396 }
2397 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
2398 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
2399 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
2400 }
2401 return SV;
2402 }
2403 case NEON::BI__builtin_neon_vdot_s32:
2404 case NEON::BI__builtin_neon_vdot_u32:
2405 case NEON::BI__builtin_neon_vdotq_s32:
2406 case NEON::BI__builtin_neon_vdotq_u32: {
2407 auto *InputTy =
2408 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
2409 llvm::Type *Tys[2] = { Ty, InputTy };
2410 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vdot");
2411 }
2412 case NEON::BI__builtin_neon_vfmlal_low_f16:
2413 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
2414 auto *InputTy =
2415 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
2416 llvm::Type *Tys[2] = { Ty, InputTy };
2417 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_low");
2418 }
2419 case NEON::BI__builtin_neon_vfmlsl_low_f16:
2420 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
2421 auto *InputTy =
2422 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
2423 llvm::Type *Tys[2] = { Ty, InputTy };
2424 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_low");
2425 }
2426 case NEON::BI__builtin_neon_vfmlal_high_f16:
2427 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
2428 auto *InputTy =
2429 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
2430 llvm::Type *Tys[2] = { Ty, InputTy };
2431 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_high");
2432 }
2433 case NEON::BI__builtin_neon_vfmlsl_high_f16:
2434 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
2435 auto *InputTy =
2436 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
2437 llvm::Type *Tys[2] = { Ty, InputTy };
2438 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_high");
2439 }
2440 case NEON::BI__builtin_neon_vmmlaq_s32:
2441 case NEON::BI__builtin_neon_vmmlaq_u32: {
2442 auto *InputTy =
2443 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
2444 llvm::Type *Tys[2] = { Ty, InputTy };
2445 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vmmla");
2446 }
2447 case NEON::BI__builtin_neon_vusmmlaq_s32: {
2448 auto *InputTy =
2449 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
2450 llvm::Type *Tys[2] = { Ty, InputTy };
2451 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusmmla");
2452 }
2453 case NEON::BI__builtin_neon_vusdot_s32:
2454 case NEON::BI__builtin_neon_vusdotq_s32: {
2455 auto *InputTy =
2456 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
2457 llvm::Type *Tys[2] = { Ty, InputTy };
2458 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusdot");
2459 }
2460 case NEON::BI__builtin_neon_vbfdot_f32:
2461 case NEON::BI__builtin_neon_vbfdotq_f32: {
2462 llvm::Type *InputTy =
2463 llvm::FixedVectorType::get(BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
2464 llvm::Type *Tys[2] = { Ty, InputTy };
2465 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vbfdot");
2466 }
2467 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
2468 llvm::Type *Tys[1] = { Ty };
2469 Function *F = CGM.getIntrinsic(Int, Tys);
2471 }
2472
2473 }
2474
2475 assert(Int && "Expected valid intrinsic number");
2476
2477
2479
2482
2483
2484 return Builder.CreateBitCast(Result, ResultType, NameHint);
2485}
2486
2489 const CmpInst::Predicate Pred,
2490 const Twine &Name) {
2491
2493
2494 Op = Builder.CreateBitCast(Op, Ty);
2495 }
2496
2497 if (CmpInst::isFPPredicate(Pred)) {
2498 if (Pred == CmpInst::FCMP_OEQ)
2499 Op = Builder.CreateFCmp(Pred, Op, Constant::getNullValue(Op->getType()));
2500 else
2501 Op = Builder.CreateFCmpS(Pred, Op, Constant::getNullValue(Op->getType()));
2502 } else {
2503 Op = Builder.CreateICmp(Pred, Op, Constant::getNullValue(Op->getType()));
2504 }
2505
2506 llvm::Type *ResTy = Ty;
2507 if (auto *VTy = dyn_cast(Ty))
2508 ResTy = FixedVectorType::get(
2509 IntegerType::get(getLLVMContext(), VTy->getScalarSizeInBits()),
2510 VTy->getNumElements());
2511
2512 return Builder.CreateSExt(Op, ResTy, Name);
2513}
2514
2517 llvm::Type *ResTy, unsigned IntID,
2518 const char *Name) {
2520 if (ExtOp)
2521 TblOps.push_back(ExtOp);
2522
2523
2526 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
2527 Indices.push_back(2*i);
2528 Indices.push_back(2*i+1);
2529 }
2530
2531 int PairPos = 0, End = Ops.size() - 1;
2532 while (PairPos < End) {
2533 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
2534 Ops[PairPos+1], Indices,
2535 Name));
2536 PairPos += 2;
2537 }
2538
2539
2540
2541 if (PairPos == End) {
2542 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
2543 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
2544 ZeroTbl, Indices, Name));
2545 }
2546
2547 Function *TblF;
2548 TblOps.push_back(IndexOp);
2550
2551 return CGF.EmitNeonCall(TblF, TblOps, Name);
2552}
2553
2554Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
2556 switch (BuiltinID) {
2557 default:
2558 return nullptr;
2559 case clang::ARM::BI__builtin_arm_nop:
2561 break;
2562 case clang::ARM::BI__builtin_arm_yield:
2563 case clang::ARM::BI__yield:
2565 break;
2566 case clang::ARM::BI__builtin_arm_wfe:
2567 case clang::ARM::BI__wfe:
2569 break;
2570 case clang::ARM::BI__builtin_arm_wfi:
2571 case clang::ARM::BI__wfi:
2573 break;
2574 case clang::ARM::BI__builtin_arm_sev:
2575 case clang::ARM::BI__sev:
2577 break;
2578 case clang::ARM::BI__builtin_arm_sevl:
2579 case clang::ARM::BI__sevl:
2581 break;
2582 }
2583
2584 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
2586}
2587
2593
2594
2595
2596
2600 llvm::Type *ValueType,
2602 StringRef SysReg = "") {
2603
2606 "Unsupported size for register.");
2607
2611
2612 if (SysReg.empty()) {
2615 }
2616
2617 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
2618 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
2619 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
2620
2622
2623 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
2624 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
2625 && "Can't fit 64-bit value in 32-bit register");
2626
2627 if (AccessKind != Write) {
2630 AccessKind == VolatileRead ? Intrinsic::read_volatile_register
2631 : Intrinsic::read_register,
2632 Types);
2633 llvm::Value *Call = Builder.CreateCall(F, Metadata);
2634
2635 if (MixedTypes)
2636
2637 return Builder.CreateTrunc(Call, ValueType);
2638
2639 if (ValueType->isPointerTy())
2640
2641 return Builder.CreateIntToPtr(Call, ValueType);
2642
2643 return Call;
2644 }
2645
2646 llvm::Function *F = CGM.getIntrinsic(Intrinsic::write_register, Types);
2648 if (MixedTypes) {
2649
2650 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
2651 return Builder.CreateCall(F, { Metadata, ArgValue });
2652 }
2653
2654 if (ValueType->isPointerTy()) {
2655
2656 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
2657 return Builder.CreateCall(F, { Metadata, ArgValue });
2658 }
2659
2660 return Builder.CreateCall(F, { Metadata, ArgValue });
2661}
2662
2663
2664
2666 switch (BuiltinID) {
2667 default: break;
2668 case NEON::BI__builtin_neon_vget_lane_i8:
2669 case NEON::BI__builtin_neon_vget_lane_i16:
2670 case NEON::BI__builtin_neon_vget_lane_bf16:
2671 case NEON::BI__builtin_neon_vget_lane_i32:
2672 case NEON::BI__builtin_neon_vget_lane_i64:
2673 case NEON::BI__builtin_neon_vget_lane_mf8:
2674 case NEON::BI__builtin_neon_vget_lane_f32:
2675 case NEON::BI__builtin_neon_vgetq_lane_i8:
2676 case NEON::BI__builtin_neon_vgetq_lane_i16:
2677 case NEON::BI__builtin_neon_vgetq_lane_bf16:
2678 case NEON::BI__builtin_neon_vgetq_lane_i32:
2679 case NEON::BI__builtin_neon_vgetq_lane_i64:
2680 case NEON::BI__builtin_neon_vgetq_lane_mf8:
2681 case NEON::BI__builtin_neon_vgetq_lane_f32:
2682 case NEON::BI__builtin_neon_vduph_lane_bf16:
2683 case NEON::BI__builtin_neon_vduph_laneq_bf16:
2684 case NEON::BI__builtin_neon_vset_lane_i8:
2685 case NEON::BI__builtin_neon_vset_lane_mf8:
2686 case NEON::BI__builtin_neon_vset_lane_i16:
2687 case NEON::BI__builtin_neon_vset_lane_bf16:
2688 case NEON::BI__builtin_neon_vset_lane_i32:
2689 case NEON::BI__builtin_neon_vset_lane_i64:
2690 case NEON::BI__builtin_neon_vset_lane_f32:
2691 case NEON::BI__builtin_neon_vsetq_lane_i8:
2692 case NEON::BI__builtin_neon_vsetq_lane_mf8:
2693 case NEON::BI__builtin_neon_vsetq_lane_i16:
2694 case NEON::BI__builtin_neon_vsetq_lane_bf16:
2695 case NEON::BI__builtin_neon_vsetq_lane_i32:
2696 case NEON::BI__builtin_neon_vsetq_lane_i64:
2697 case NEON::BI__builtin_neon_vsetq_lane_f32:
2698 case NEON::BI__builtin_neon_vsha1h_u32:
2699 case NEON::BI__builtin_neon_vsha1cq_u32:
2700 case NEON::BI__builtin_neon_vsha1pq_u32:
2701 case NEON::BI__builtin_neon_vsha1mq_u32:
2702 case NEON::BI__builtin_neon_vcvth_bf16_f32:
2703 case clang::ARM::BI_MoveToCoprocessor:
2704 case clang::ARM::BI_MoveToCoprocessor2:
2705 return false;
2706 }
2707 return true;
2708}
2709
2713 llvm::Triple::ArchType Arch) {
2714 if (auto Hint = GetValueForARMHint(BuiltinID))
2715 return Hint;
2716
2717 if (BuiltinID == clang::ARM::BI__emit) {
2718 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
2719 llvm::FunctionType *FTy =
2720 llvm::FunctionType::get(VoidTy, false);
2721
2724 llvm_unreachable("Sema will ensure that the parameter is constant");
2725
2726 llvm::APSInt Value = Result.Val.getInt();
2727 uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
2728
2729 llvm::InlineAsm *Emit =
2730 IsThumb ? InlineAsm::get(FTy, ".inst.n 0x" + utohexstr(ZExtValue), "",
2731 true)
2732 : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "",
2733 true);
2734
2735 return Builder.CreateCall(Emit);
2736 }
2737
2738 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
2740 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
2741 }
2742
2743 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
2747
2748
2749 Value *Locality = llvm::ConstantInt::get(Int32Ty, 3);
2750
2752 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
2753 }
2754
2755 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
2757 return Builder.CreateCall(
2758 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
2759 }
2760
2761 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
2762 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
2764 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
2766 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
2767 Res = Builder.CreateTrunc(Res, Builder.getInt32Ty());
2768 return Res;
2769 }
2770
2771
2772 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
2774 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls), Arg, "cls");
2775 }
2776 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
2778 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls64), Arg,
2779 "cls");
2780 }
2781
2782 if (BuiltinID == clang::ARM::BI__clear_cache) {
2783 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
2786 for (unsigned i = 0; i < 2; i++)
2788 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
2790 StringRef Name = FD->getName();
2792 }
2793
2794 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
2795 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
2797
2798 switch (BuiltinID) {
2799 default: llvm_unreachable("unexpected builtin");
2800 case clang::ARM::BI__builtin_arm_mcrr:
2801 F = CGM.getIntrinsic(Intrinsic::arm_mcrr);
2802 break;
2803 case clang::ARM::BI__builtin_arm_mcrr2:
2804 F = CGM.getIntrinsic(Intrinsic::arm_mcrr2);
2805 break;
2806 }
2807
2808
2809
2810
2811
2812
2813
2814
2819
2820 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
2822 Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
2824
2825 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
2826 }
2827
2828 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
2829 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
2831
2832 switch (BuiltinID) {
2833 default: llvm_unreachable("unexpected builtin");
2834 case clang::ARM::BI__builtin_arm_mrrc:
2835 F = CGM.getIntrinsic(Intrinsic::arm_mrrc);
2836 break;
2837 case clang::ARM::BI__builtin_arm_mrrc2:
2838 F = CGM.getIntrinsic(Intrinsic::arm_mrrc2);
2839 break;
2840 }
2841
2845 Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
2846
2847
2848
2849
2850 Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
2851 Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
2854
2855 Value *ShiftCast = llvm::ConstantInt::get(Int64Ty, 32);
2856 RtAndRt2 = Builder.CreateShl(Rt, ShiftCast, "shl", true);
2857 RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
2858
2860 }
2861
2862 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
2863 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2864 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
2866 BuiltinID == clang::ARM::BI__ldrexd) {
2868
2869 switch (BuiltinID) {
2870 default: llvm_unreachable("unexpected builtin");
2871 case clang::ARM::BI__builtin_arm_ldaex:
2872 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
2873 break;
2874 case clang::ARM::BI__builtin_arm_ldrexd:
2875 case clang::ARM::BI__builtin_arm_ldrex:
2876 case clang::ARM::BI__ldrexd:
2877 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
2878 break;
2879 }
2880
2882 Value *Val = Builder.CreateCall(F, LdPtr, "ldrexd");
2883
2884 Value *Val0 = Builder.CreateExtractValue(Val, 1);
2885 Value *Val1 = Builder.CreateExtractValue(Val, 0);
2888
2889 Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
2890 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true );
2891 Val = Builder.CreateOr(Val, Val1);
2893 }
2894
2895 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
2896 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
2898
2900 llvm::Type *RealResTy = ConvertType(Ty);
2901 llvm::Type *IntTy =
2903
2905 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
2906 : Intrinsic::arm_ldrex,
2908 CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldrex");
2909 Val->addParamAttr(
2911
2912 if (RealResTy->isPointerTy())
2913 return Builder.CreateIntToPtr(Val, RealResTy);
2914 else {
2915 llvm::Type *IntResTy = llvm::IntegerType::get(
2916 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
2917 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
2918 RealResTy);
2919 }
2920 }
2921
2922 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
2923 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
2924 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
2927 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
2928 : Intrinsic::arm_strexd);
2929 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty);
2930
2933 Builder.CreateStore(Val, Tmp);
2934
2936 Val = Builder.CreateLoad(LdPtr);
2937
2938 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
2939 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
2941 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd");
2942 }
2943
2944 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
2945 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
2948
2950 llvm::Type *StoreTy =
2952
2953 if (StoreVal->getType()->isPointerTy())
2954 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
2955 else {
2956 llvm::Type *IntTy = llvm::IntegerType::get(
2958 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
2959 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
2960 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
2961 }
2962
2964 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
2965 : Intrinsic::arm_strex,
2967
2968 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex");
2969 CI->addParamAttr(
2970 1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
2971 return CI;
2972 }
2973
2974 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
2975 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
2976 return Builder.CreateCall(F);
2977 }
2978
2979
2980 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
2981 switch (BuiltinID) {
2982 case clang::ARM::BI__builtin_arm_crc32b:
2983 CRCIntrinsicID = Intrinsic::arm_crc32b; break;
2984 case clang::ARM::BI__builtin_arm_crc32cb:
2985 CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
2986 case clang::ARM::BI__builtin_arm_crc32h:
2987 CRCIntrinsicID = Intrinsic::arm_crc32h; break;
2988 case clang::ARM::BI__builtin_arm_crc32ch:
2989 CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
2990 case clang::ARM::BI__builtin_arm_crc32w:
2991 case clang::ARM::BI__builtin_arm_crc32d:
2992 CRCIntrinsicID = Intrinsic::arm_crc32w; break;
2993 case clang::ARM::BI__builtin_arm_crc32cw:
2994 case clang::ARM::BI__builtin_arm_crc32cd:
2995 CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
2996 }
2997
2998 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
3001
3002
3003
3004 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
3005 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
3006 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
3008 Value *Arg1b = Builder.CreateLShr(Arg1, C1);
3009 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
3010
3011 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
3012 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
3013 return Builder.CreateCall(F, {Res, Arg1b});
3014 } else {
3016
3017 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
3018 return Builder.CreateCall(F, {Arg0, Arg1});
3019 }
3020 }
3021
3022 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
3023 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
3024 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
3025 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
3026 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
3027 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
3028
3030 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
3031 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
3032 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
3034
3035 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
3036 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
3037
3038 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
3039 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
3040
3041 llvm::Type *ValueType;
3043 if (IsPointerBuiltin) {
3046 } else if (Is64Bit) {
3048 } else {
3050 }
3051
3053 AccessKind);
3054 }
3055
3056 if (BuiltinID == ARM::BI__builtin_sponentry) {
3057 llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
3058 return Builder.CreateCall(F);
3059 }
3060
3061
3062
3065
3066
3069
3072
3073
3075 return P.first == BuiltinID;
3076 });
3078 BuiltinID = It->second;
3079
3080
3081
3082 unsigned ICEArguments = 0;
3086
3087 auto getAlignmentValue32 = [&](Address addr) -> Value* {
3088 return Builder.getInt32(addr.getAlignment().getQuantity());
3089 };
3090
3095 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
3096 for (unsigned i = 0, e = NumArgs; i != e; i++) {
3097 if (i == 0) {
3098 switch (BuiltinID) {
3099 case NEON::BI__builtin_neon_vld1_v:
3100 case NEON::BI__builtin_neon_vld1q_v:
3101 case NEON::BI__builtin_neon_vld1q_lane_v:
3102 case NEON::BI__builtin_neon_vld1_lane_v:
3103 case NEON::BI__builtin_neon_vld1_dup_v:
3104 case NEON::BI__builtin_neon_vld1q_dup_v:
3105 case NEON::BI__builtin_neon_vst1_v:
3106 case NEON::BI__builtin_neon_vst1q_v:
3107 case NEON::BI__builtin_neon_vst1q_lane_v:
3108 case NEON::BI__builtin_neon_vst1_lane_v:
3109 case NEON::BI__builtin_neon_vst2_v:
3110 case NEON::BI__builtin_neon_vst2q_v:
3111 case NEON::BI__builtin_neon_vst2_lane_v:
3112 case NEON::BI__builtin_neon_vst2q_lane_v:
3113 case NEON::BI__builtin_neon_vst3_v:
3114 case NEON::BI__builtin_neon_vst3q_v:
3115 case NEON::BI__builtin_neon_vst3_lane_v:
3116 case NEON::BI__builtin_neon_vst3q_lane_v:
3117 case NEON::BI__builtin_neon_vst4_v:
3118 case NEON::BI__builtin_neon_vst4q_v:
3119 case NEON::BI__builtin_neon_vst4_lane_v:
3120 case NEON::BI__builtin_neon_vst4q_lane_v:
3121
3122
3125 continue;
3126 }
3127 }
3128 if (i == 1) {
3129 switch (BuiltinID) {
3130 case NEON::BI__builtin_neon_vld2_v:
3131 case NEON::BI__builtin_neon_vld2q_v:
3132 case NEON::BI__builtin_neon_vld3_v:
3133 case NEON::BI__builtin_neon_vld3q_v:
3134 case NEON::BI__builtin_neon_vld4_v:
3135 case NEON::BI__builtin_neon_vld4q_v:
3136 case NEON::BI__builtin_neon_vld2_lane_v:
3137 case NEON::BI__builtin_neon_vld2q_lane_v:
3138 case NEON::BI__builtin_neon_vld3_lane_v:
3139 case NEON::BI__builtin_neon_vld3q_lane_v:
3140 case NEON::BI__builtin_neon_vld4_lane_v:
3141 case NEON::BI__builtin_neon_vld4q_lane_v:
3142 case NEON::BI__builtin_neon_vld2_dup_v:
3143 case NEON::BI__builtin_neon_vld2q_dup_v:
3144 case NEON::BI__builtin_neon_vld3_dup_v:
3145 case NEON::BI__builtin_neon_vld3q_dup_v:
3146 case NEON::BI__builtin_neon_vld4_dup_v:
3147 case NEON::BI__builtin_neon_vld4q_dup_v:
3148
3149
3152 continue;
3153 }
3154 }
3155
3157 }
3158
3159 switch (BuiltinID) {
3160 default: break;
3161
3162 case NEON::BI__builtin_neon_vget_lane_i8:
3163 case NEON::BI__builtin_neon_vget_lane_i16:
3164 case NEON::BI__builtin_neon_vget_lane_i32:
3165 case NEON::BI__builtin_neon_vget_lane_i64:
3166 case NEON::BI__builtin_neon_vget_lane_bf16:
3167 case NEON::BI__builtin_neon_vget_lane_f32:
3168 case NEON::BI__builtin_neon_vgetq_lane_i8:
3169 case NEON::BI__builtin_neon_vgetq_lane_i16:
3170 case NEON::BI__builtin_neon_vgetq_lane_i32:
3171 case NEON::BI__builtin_neon_vgetq_lane_i64:
3172 case NEON::BI__builtin_neon_vgetq_lane_bf16:
3173 case NEON::BI__builtin_neon_vgetq_lane_f32:
3174 case NEON::BI__builtin_neon_vduph_lane_bf16:
3175 case NEON::BI__builtin_neon_vduph_laneq_bf16:
3176 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
3177
3178 case NEON::BI__builtin_neon_vrndns_f32: {
3180 llvm::Type *Tys[] = {Arg->getType()};
3181 Function *F = CGM.getIntrinsic(Intrinsic::roundeven, Tys);
3182 return Builder.CreateCall(F, {Arg}, "vrndn"); }
3183
3184 case NEON::BI__builtin_neon_vset_lane_i8:
3185 case NEON::BI__builtin_neon_vset_lane_i16:
3186 case NEON::BI__builtin_neon_vset_lane_i32:
3187 case NEON::BI__builtin_neon_vset_lane_i64:
3188 case NEON::BI__builtin_neon_vset_lane_bf16:
3189 case NEON::BI__builtin_neon_vset_lane_f32:
3190 case NEON::BI__builtin_neon_vsetq_lane_i8:
3191 case NEON::BI__builtin_neon_vsetq_lane_i16:
3192 case NEON::BI__builtin_neon_vsetq_lane_i32:
3193 case NEON::BI__builtin_neon_vsetq_lane_i64:
3194 case NEON::BI__builtin_neon_vsetq_lane_bf16:
3195 case NEON::BI__builtin_neon_vsetq_lane_f32:
3196 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
3197
3198 case NEON::BI__builtin_neon_vsha1h_u32:
3199 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
3200 "vsha1h");
3201 case NEON::BI__builtin_neon_vsha1cq_u32:
3202 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
3203 "vsha1h");
3204 case NEON::BI__builtin_neon_vsha1pq_u32:
3205 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
3206 "vsha1h");
3207 case NEON::BI__builtin_neon_vsha1mq_u32:
3208 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
3209 "vsha1h");
3210
3211 case NEON::BI__builtin_neon_vcvth_bf16_f32: {
3212 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtbfp2bf), Ops,
3213 "vcvtbfp2bf");
3214 }
3215
3216
3217
3218 case clang::ARM::BI_MoveToCoprocessor:
3219 case clang::ARM::BI_MoveToCoprocessor2: {
3220 Function *F = CGM.getIntrinsic(BuiltinID == clang::ARM::BI_MoveToCoprocessor
3221 ? Intrinsic::arm_mcr
3222 : Intrinsic::arm_mcr2);
3223 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
3224 Ops[3], Ops[4], Ops[5]});
3225 }
3226 }
3227
3228
3229 assert(HasExtraArg);
3231 std::optionalllvm::APSInt Result =
3234 return nullptr;
3235
3236 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
3237 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
3238
3239 llvm::Type *Ty;
3240 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
3242 else
3244
3245
3246 bool usgn = Result->getZExtValue() == 1;
3247 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
3248
3249
3250 Function *F = CGM.getIntrinsic(Int, Ty);
3251 return Builder.CreateCall(F, Ops, "vcvtr");
3252 }
3253
3254
3256 bool usgn = Type.isUnsigned();
3257 bool rightShift = false;
3258
3259 llvm::FixedVectorType *VTy =
3262 llvm::Type *Ty = VTy;
3263 if (!Ty)
3264 return nullptr;
3265
3266
3267
3274 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
3275
3276 unsigned Int;
3277 switch (BuiltinID) {
3278 default: return nullptr;
3279 case NEON::BI__builtin_neon_vld1q_lane_v:
3280
3281
3282 if (VTy->getElementType()->isIntegerTy(64)) {
3283
3284 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3286 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
3287 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
3288
3289 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
3290 llvm::Type *Tys[] = {Ty, Int8PtrTy};
3291 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
3292 Value *Align = getAlignmentValue32(PtrOp0);
3293 Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
3294
3295 int Indices[] = {1 - Lane, Lane};
3296 return Builder.CreateShuffleVector(Ops[1], Ld, Indices, "vld1q_lane");
3297 }
3298 [[fallthrough]];
3299 case NEON::BI__builtin_neon_vld1_lane_v: {
3300 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3303 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
3304 }
3305 case NEON::BI__builtin_neon_vqrshrn_n_v:
3306 Int =
3307 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
3308 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n",
3309 1, true);
3310 case NEON::BI__builtin_neon_vqrshrun_n_v:
3311 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
3312 Ops, "vqrshrun_n", 1, true);
3313 case NEON::BI__builtin_neon_vqshrn_n_v:
3314 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
3315 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n",
3316 1, true);
3317 case NEON::BI__builtin_neon_vqshrun_n_v:
3318 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
3319 Ops, "vqshrun_n", 1, true);
3320 case NEON::BI__builtin_neon_vrecpe_v:
3321 case NEON::BI__builtin_neon_vrecpeq_v:
3322 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
3323 Ops, "vrecpe");
3324 case NEON::BI__builtin_neon_vrshrn_n_v:
3325 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
3326 Ops, "vrshrn_n", 1, true);
3327 case NEON::BI__builtin_neon_vrsra_n_v:
3328 case NEON::BI__builtin_neon_vrsraq_n_v:
3329 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3330 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3332 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
3333 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
3334 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
3335 case NEON::BI__builtin_neon_vsri_n_v:
3336 case NEON::BI__builtin_neon_vsriq_n_v:
3337 rightShift = true;
3338 [[fallthrough]];
3339 case NEON::BI__builtin_neon_vsli_n_v:
3340 case NEON::BI__builtin_neon_vsliq_n_v:
3342 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
3343 Ops, "vsli_n");
3344 case NEON::BI__builtin_neon_vsra_n_v:
3345 case NEON::BI__builtin_neon_vsraq_n_v:
3346 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3348 return Builder.CreateAdd(Ops[0], Ops[1]);
3349 case NEON::BI__builtin_neon_vst1q_lane_v:
3350
3351
3352 if (VTy->getElementType()->isIntegerTy(64)) {
3353 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3355 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
3356 Ops[2] = getAlignmentValue32(PtrOp0);
3357 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
3358 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
3359 Tys), Ops);
3360 }
3361 [[fallthrough]];
3362 case NEON::BI__builtin_neon_vst1_lane_v: {
3363 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3364 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
3365 return Builder.CreateStore(Ops[1],
3367 }
3368 case NEON::BI__builtin_neon_vtbl1_v:
3369 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
3370 Ops, "vtbl1");
3371 case NEON::BI__builtin_neon_vtbl2_v:
3372 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
3373 Ops, "vtbl2");
3374 case NEON::BI__builtin_neon_vtbl3_v:
3375 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
3376 Ops, "vtbl3");
3377 case NEON::BI__builtin_neon_vtbl4_v:
3378 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
3379 Ops, "vtbl4");
3380 case NEON::BI__builtin_neon_vtbx1_v:
3381 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
3382 Ops, "vtbx1");
3383 case NEON::BI__builtin_neon_vtbx2_v:
3384 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
3385 Ops, "vtbx2");
3386 case NEON::BI__builtin_neon_vtbx3_v:
3387 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
3388 Ops, "vtbx3");
3389 case NEON::BI__builtin_neon_vtbx4_v:
3390 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
3391 Ops, "vtbx4");
3392 }
3393}
3394
3395template
3399
3402
3403
3404 return Unsigned ? Builder.CreateZExt(V, T) : Builder.CreateSExt(V, T);
3405}
3406
3408 uint32_t Shift, bool Unsigned) {
3409
3410
3411
3412
3413
3415 ->getElementType()
3416 ->getPrimitiveSizeInBits();
3417 if (Shift == LaneBits) {
3418
3419
3420
3422 return llvm::Constant::getNullValue(V->getType());
3423 else
3424 --Shift;
3425 }
3426 return Unsigned ? Builder.CreateLShr(V, Shift) : Builder.CreateAShr(V, Shift);
3427}
3428
3430
3431
3432
3433 unsigned Elements = 128 / V->getType()->getPrimitiveSizeInBits();
3434 return Builder.CreateVectorSplat(Elements, V);
3435}
3436
3439 llvm::Value *V,
3440 llvm::Type *DestType) {
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3453 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
3454 return Builder.CreateCall(
3456 {DestType, V->getType()}),
3457 V);
3458 } else {
3459 return Builder.CreateBitCast(V, DestType);
3460 }
3461}
3462
3464
3465
3467 unsigned InputElements =
3469 for (unsigned i = 0; i < InputElements; i += 2)
3470 Indices.push_back(i + Odd);
3471 return Builder.CreateShuffleVector(V, Indices);
3472}
3473
3475 llvm::Value *V1) {
3476
3477 assert(V0->getType() == V1->getType() && "Can't zip different vector types");
3479 unsigned InputElements =
3481 for (unsigned i = 0; i < InputElements; i++) {
3482 Indices.push_back(i);
3483 Indices.push_back(i + InputElements);
3484 }
3485 return Builder.CreateShuffleVector(V0, V1, Indices);
3486}
3487
3488template<unsigned HighBit, unsigned OtherBits>
3490
3491
3493 unsigned LaneBits = T->getPrimitiveSizeInBits();
3494 uint32_t Value = HighBit << (LaneBits - 1);
3495 if (OtherBits)
3496 Value |= (1UL << (LaneBits - 1)) - 1;
3497 llvm::Value *Lane = llvm::ConstantInt::get(T, Value);
3499}
3500
3502 llvm::Value *V,
3503 unsigned ReverseWidth) {
3504
3505
3507 unsigned LaneSize = V->getType()->getScalarSizeInBits();
3508 unsigned Elements = 128 / LaneSize;
3509 unsigned Mask = ReverseWidth / LaneSize - 1;
3510 for (unsigned i = 0; i < Elements; i++)
3511 Indices.push_back(i ^ Mask);
3512 return Builder.CreateShuffleVector(V, Indices);
3513}
3514
3517 llvm::Type *Ty) {
3518 return Builder.CreateCall(
3519 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vcvt_fp_int, {Ty, V->getType()}),
3520 {V, llvm::ConstantInt::get(Builder.getInt32Ty(), 0)});
3521}
3522
3525 llvm::Type *Ty) {
3526 return Builder.CreateCall(
3527 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vcvt_fp_int, {Ty, V->getType()}),
3528 {V, llvm::ConstantInt::get(Builder.getInt32Ty(), 1)});
3529}
3530
3533 llvm::Type *Ty) {
3534 return Builder.CreateCall(
3535 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vcvt_int_fp, {Ty, V->getType()}),
3536 {V, llvm::ConstantInt::get(Builder.getInt32Ty(), 0)});
3537}
3538
3541 llvm::Type *Ty) {
3542 return Builder.CreateCall(
3543 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vcvt_int_fp, {Ty, V->getType()}),
3544 {V, llvm::ConstantInt::get(Builder.getInt32Ty(), 1)});
3545}
3546
3550 llvm::Triple::ArchType Arch) {
3551 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
3552 Intrinsic::ID IRIntr;
3553 unsigned NumVectors;
3554
3555
3556 switch (BuiltinID) {
3557 #include "clang/Basic/arm_mve_builtin_cg.inc"
3558
3559
3560
3561 default:
3562 return nullptr;
3563 }
3564
3565
3566
3567
3568 switch (CustomCodeGenType) {
3569
3570 case CustomCodeGen::VLD24: {
3573
3574 auto MvecCType = E->getType();
3575 auto MvecLType = ConvertType(MvecCType);
3576 assert(MvecLType->isStructTy() &&
3577 "Return type for vld[24]q should be a struct");
3578 assert(MvecLType->getStructNumElements() == 1 &&
3579 "Return-type struct for vld[24]q should have one element");
3580 auto MvecLTypeInner = MvecLType->getStructElementType(0);
3581 assert(MvecLTypeInner->isArrayTy() &&
3582 "Return-type struct for vld[24]q should contain an array");
3583 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
3584 "Array member of return-type struct vld[24]q has wrong length");
3585 auto VecLType = MvecLTypeInner->getArrayElementType();
3586
3587 Tys.push_back(VecLType);
3588
3592
3594 Value *LoadResult = Builder.CreateCall(F, Ops);
3595 Value *MvecOut = PoisonValue::get(MvecLType);
3596 for (unsigned i = 0; i < NumVectors; ++i) {
3597 Value *Vec = Builder.CreateExtractValue(LoadResult, i);
3598 MvecOut = Builder.CreateInsertValue(MvecOut, Vec, {0, i});
3599 }
3600
3602 return MvecOut;
3603 else
3605 }
3606
3607 case CustomCodeGen::VST24: {
3610
3614
3616 auto MvecLType = ConvertType(MvecCType);
3617 assert(MvecLType->isStructTy() && "Data type for vst2q should be a struct");
3618 assert(MvecLType->getStructNumElements() == 1 &&
3619 "Data-type struct for vst2q should have one element");
3620 auto MvecLTypeInner = MvecLType->getStructElementType(0);
3621 assert(MvecLTypeInner->isArrayTy() &&
3622 "Data-type struct for vst2q should contain an array");
3623 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
3624 "Array member of return-type struct vld[24]q has wrong length");
3625 auto VecLType = MvecLTypeInner->getArrayElementType();
3626
3627 Tys.push_back(VecLType);
3628
3632 for (unsigned i = 0; i < NumVectors; i++)
3633 Ops.push_back(Builder.CreateExtractValue(Mvec, {0, i}));
3634
3636 Value *ToReturn = nullptr;
3637 for (unsigned i = 0; i < NumVectors; i++) {
3638 Ops.push_back(llvm::ConstantInt::get(Int32Ty, i));
3639 ToReturn = Builder.CreateCall(F, Ops);
3640 Ops.pop_back();
3641 }
3642 return ToReturn;
3643 }
3644 }
3645 llvm_unreachable("unknown custom codegen type.");
3646}
3647
3651 llvm::Triple::ArchType Arch) {
3652 switch (BuiltinID) {
3653 default:
3654 return nullptr;
3655#include "clang/Basic/arm_cde_builtin_cg.inc"
3656 }
3657}
3658
3662 llvm::Triple::ArchType Arch) {
3663 unsigned int Int = 0;
3664 const char *s = nullptr;
3665
3666 switch (BuiltinID) {
3667 default:
3668 return nullptr;
3669 case NEON::BI__builtin_neon_vtbl1_v:
3670 case NEON::BI__builtin_neon_vqtbl1_v:
3671 case NEON::BI__builtin_neon_vqtbl1q_v:
3672 case NEON::BI__builtin_neon_vtbl2_v:
3673 case NEON::BI__builtin_neon_vqtbl2_v:
3674 case NEON::BI__builtin_neon_vqtbl2q_v:
3675 case NEON::BI__builtin_neon_vtbl3_v:
3676 case NEON::BI__builtin_neon_vqtbl3_v:
3677 case NEON::BI__builtin_neon_vqtbl3q_v:
3678 case NEON::BI__builtin_neon_vtbl4_v:
3679 case NEON::BI__builtin_neon_vqtbl4_v:
3680 case NEON::BI__builtin_neon_vqtbl4q_v:
3681 break;
3682 case NEON::BI__builtin_neon_vtbx1_v:
3683 case NEON::BI__builtin_neon_vqtbx1_v:
3684 case NEON::BI__builtin_neon_vqtbx1q_v:
3685 case NEON::BI__builtin_neon_vtbx2_v:
3686 case NEON::BI__builtin_neon_vqtbx2_v:
3687 case NEON::BI__builtin_neon_vqtbx2q_v:
3688 case NEON::BI__builtin_neon_vtbx3_v:
3689 case NEON::BI__builtin_neon_vqtbx3_v:
3690 case NEON::BI__builtin_neon_vqtbx3q_v:
3691 case NEON::BI__builtin_neon_vtbx4_v:
3692 case NEON::BI__builtin_neon_vqtbx4_v:
3693 case NEON::BI__builtin_neon_vqtbx4q_v:
3694 break;
3695 }
3696
3698
3699
3701 std::optionalllvm::APSInt Result =
3703 if (!Result)
3704 return nullptr;
3705
3706
3709 if (!Ty)
3710 return nullptr;
3711
3713
3714
3715
3716 switch (BuiltinID) {
3717 case NEON::BI__builtin_neon_vtbl1_v: {
3719 Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
3720 }
3721 case NEON::BI__builtin_neon_vtbl2_v: {
3723 Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
3724 }
3725 case NEON::BI__builtin_neon_vtbl3_v: {
3727 Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
3728 }
3729 case NEON::BI__builtin_neon_vtbl4_v: {
3731 Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
3732 }
3733 case NEON::BI__builtin_neon_vtbx1_v: {
3736 Intrinsic::aarch64_neon_tbl1, "vtbl1");
3737
3738 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
3739 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
3740 CmpRes = Builder.CreateSExt(CmpRes, Ty);
3741
3742 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
3743 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
3744 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
3745 }
3746 case NEON::BI__builtin_neon_vtbx2_v: {
3748 Ty, Intrinsic::aarch64_neon_tbx1, "vtbx1");
3749 }
3750 case NEON::BI__builtin_neon_vtbx3_v: {
3753 Intrinsic::aarch64_neon_tbl2, "vtbl2");
3754
3755 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
3756 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
3757 TwentyFourV);
3758 CmpRes = Builder.CreateSExt(CmpRes, Ty);
3759
3760 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
3761 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
3762 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
3763 }
3764 case NEON::BI__builtin_neon_vtbx4_v: {
3766 Ty, Intrinsic::aarch64_neon_tbx2, "vtbx2");
3767 }
3768 case NEON::BI__builtin_neon_vqtbl1_v:
3769 case NEON::BI__builtin_neon_vqtbl1q_v:
3770 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break;
3771 case NEON::BI__builtin_neon_vqtbl2_v:
3772 case NEON::BI__builtin_neon_vqtbl2q_v: {
3773 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break;
3774 case NEON::BI__builtin_neon_vqtbl3_v:
3775 case NEON::BI__builtin_neon_vqtbl3q_v:
3776 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break;
3777 case NEON::BI__builtin_neon_vqtbl4_v:
3778 case NEON::BI__builtin_neon_vqtbl4q_v:
3779 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break;
3780 case NEON::BI__builtin_neon_vqtbx1_v:
3781 case NEON::BI__builtin_neon_vqtbx1q_v:
3782 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break;
3783 case NEON::BI__builtin_neon_vqtbx2_v:
3784 case NEON::BI__builtin_neon_vqtbx2q_v:
3785 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break;
3786 case NEON::BI__builtin_neon_vqtbx3_v:
3787 case NEON::BI__builtin_neon_vqtbx3q_v:
3788 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break;
3789 case NEON::BI__builtin_neon_vqtbx4_v:
3790 case NEON::BI__builtin_neon_vqtbx4q_v:
3791 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break;
3792 }
3793 }
3794
3795 if (!Int)
3796 return nullptr;
3797
3800}
3801
3803 auto *VTy = llvm::FixedVectorType::get(Int16Ty, 4);
3805 Value *V = PoisonValue::get(VTy);
3806 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
3807 Op = Builder.CreateInsertElement(V, Op, CI);
3808 return Op;
3809}
3810
3811
3812
3813
3816 case SVETypeFlags::MemEltTyDefault:
3818 case SVETypeFlags::MemEltTyInt8:
3819 return Builder.getInt8Ty();
3820 case SVETypeFlags::MemEltTyInt16:
3821 return Builder.getInt16Ty();
3822 case SVETypeFlags::MemEltTyInt32:
3823 return Builder.getInt32Ty();
3824 case SVETypeFlags::MemEltTyInt64:
3825 return Builder.getInt64Ty();
3826 }
3827 llvm_unreachable("Unknown MemEltType");
3828}
3829
3832 default:
3833 llvm_unreachable("Invalid SVETypeFlag!");
3834
3835 case SVETypeFlags::EltTyMFloat8:
3836 case SVETypeFlags::EltTyInt8:
3837 return Builder.getInt8Ty();
3838 case SVETypeFlags::EltTyInt16:
3839 return Builder.getInt16Ty();
3840 case SVETypeFlags::EltTyInt32:
3841 return Builder.getInt32Ty();
3842 case SVETypeFlags::EltTyInt64:
3843 return Builder.getInt64Ty();
3844 case SVETypeFlags::EltTyInt128:
3845 return Builder.getInt128Ty();
3846
3847 case SVETypeFlags::EltTyFloat16:
3848 return Builder.getHalfTy();
3849 case SVETypeFlags::EltTyFloat32:
3850 return Builder.getFloatTy();
3851 case SVETypeFlags::EltTyFloat64:
3852 return Builder.getDoubleTy();
3853
3854 case SVETypeFlags::EltTyBFloat16:
3855 return Builder.getBFloatTy();
3856
3857 case SVETypeFlags::EltTyBool8:
3858 case SVETypeFlags::EltTyBool16:
3859 case SVETypeFlags::EltTyBool32:
3860 case SVETypeFlags::EltTyBool64:
3861 return Builder.getInt1Ty();
3862 }
3863}
3864
3865
3866
3867llvm::ScalableVectorType *
3870 default: llvm_unreachable("Unhandled SVETypeFlag!");
3871
3872 case SVETypeFlags::EltTyInt8:
3873 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
3874 case SVETypeFlags::EltTyInt16:
3875 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3876 case SVETypeFlags::EltTyInt32:
3877 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
3878 case SVETypeFlags::EltTyInt64:
3879 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
3880
3881 case SVETypeFlags::EltTyBFloat16:
3882 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3883 case SVETypeFlags::EltTyFloat16:
3884 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3885 case SVETypeFlags::EltTyFloat32:
3886 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
3887 case SVETypeFlags::EltTyFloat64:
3888 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
3889
3890 case SVETypeFlags::EltTyBool8:
3891 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
3892 case SVETypeFlags::EltTyBool16:
3893 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3894 case SVETypeFlags::EltTyBool32:
3895 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
3896 case SVETypeFlags::EltTyBool64:
3897 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
3898 }
3899}
3900
3901
3902llvm::ScalableVectorType *
3905 default:
3906 llvm_unreachable("Invalid SVETypeFlag!");
3907
3908 case SVETypeFlags::EltTyInt8:
3909 return llvm::ScalableVectorType::get(Builder.getInt8Ty(), 16);
3910 case SVETypeFlags::EltTyInt16:
3911 return llvm::ScalableVectorType::get(Builder.getInt16Ty(), 8);
3912 case SVETypeFlags::EltTyInt32:
3913 return llvm::ScalableVectorType::get(Builder.getInt32Ty(), 4);
3914 case SVETypeFlags::EltTyInt64:
3915 return llvm::ScalableVectorType::get(Builder.getInt64Ty(), 2);
3916
3917 case SVETypeFlags::EltTyMFloat8:
3918 return llvm::ScalableVectorType::get(Builder.getInt8Ty(), 16);
3919 case SVETypeFlags::EltTyFloat16:
3920 return llvm::ScalableVectorType::get(Builder.getHalfTy(), 8);
3921 case SVETypeFlags::EltTyBFloat16:
3922 return llvm::ScalableVectorType::get(Builder.getBFloatTy(), 8);
3923 case SVETypeFlags::EltTyFloat32:
3924 return llvm::ScalableVectorType::get(Builder.getFloatTy(), 4);
3925 case SVETypeFlags::EltTyFloat64:
3926 return llvm::ScalableVectorType::get(Builder.getDoubleTy(), 2);
3927
3928 case SVETypeFlags::EltTyBool8:
3929 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
3930 case SVETypeFlags::EltTyBool16:
3931 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
3932 case SVETypeFlags::EltTyBool32:
3933 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
3934 case SVETypeFlags::EltTyBool64:
3935 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
3936 }
3937}
3938
3939llvm::Value *
3945
3947
3949 unsigned NumElts = SVEBitsPerBlock / EltTy->getScalarSizeInBits();
3950 return llvm::ScalableVectorType::get(EltTy, NumElts);
3951}
3952
3953
3954
3956 llvm::ScalableVectorType *VTy) {
3957
3960 return Pred;
3961
3962 auto *RTy = llvm::VectorType::get(IntegerType::get(getLLVMContext(), 1), VTy);
3963 if (Pred->getType() == RTy)
3964 return Pred;
3965
3966 unsigned IntID;
3967 llvm::Type *IntrinsicTy;
3968 switch (VTy->getMinNumElements()) {
3969 default:
3970 llvm_unreachable("unsupported element count!");
3971 case 1:
3972 case 2:
3973 case 4:
3974 case 8:
3975 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
3976 IntrinsicTy = RTy;
3977 break;
3978 case 16:
3979 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
3980 IntrinsicTy = Pred->getType();
3981 break;
3982 }
3983
3984 Function *F = CGM.getIntrinsic(IntID, IntrinsicTy);
3986 assert(C->getType() == RTy && "Unexpected return type!");
3987 return C;
3988}
3989
3991 llvm::StructType *Ty) {
3992 if (PredTuple->getType() == Ty)
3993 return PredTuple;
3994
3995 Value *Ret = llvm::PoisonValue::get(Ty);
3996 for (unsigned I = 0; I < Ty->getNumElements(); ++I) {
3997 Value *Pred = Builder.CreateExtractValue(PredTuple, I);
4000 Ret = Builder.CreateInsertValue(Ret, Pred, I);
4001 }
4002
4003 return Ret;
4004}
4005
4008 unsigned IntID) {
4009 auto *ResultTy = getSVEType(TypeFlags);
4010 auto *OverloadedTy =
4011 llvm::ScalableVectorType::get(SVEBuiltinMemEltTy(TypeFlags), ResultTy);
4012
4014 if (Ops[1]->getType()->isVectorTy())
4015
4016
4017
4018 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[1]->getType()});
4019 else
4020
4021
4022
4023
4024 F = CGM.getIntrinsic(IntID, OverloadedTy);
4025
4026
4027
4028
4029
4030
4031
4032
4035
4036
4037
4038
4039 if (Ops.size() == 2) {
4040 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
4041 Ops.push_back(ConstantInt::get(Int64Ty, 0));
4042 }
4043
4044
4045
4046 if (!TypeFlags.isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
4047 unsigned BytesPerElt =
4048 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
4049 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
4050 }
4051
4053
4054
4055
4058}
4059
4062 unsigned IntID) {
4063 auto *SrcDataTy = getSVEType(TypeFlags);
4064 auto *OverloadedTy =
4065 llvm::ScalableVectorType::get(SVEBuiltinMemEltTy(TypeFlags), SrcDataTy);
4066
4067
4068
4069 Ops.insert(Ops.begin(), Ops.pop_back_val());
4070
4072 if (Ops[2]->getType()->isVectorTy())
4073
4074
4075
4076 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[2]->getType()});
4077 else
4078
4079
4080
4081
4082 F = CGM.getIntrinsic(IntID, OverloadedTy);
4083
4084
4085
4086
4087 if (Ops.size() == 3) {
4088 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
4089 Ops.push_back(ConstantInt::get(Int64Ty, 0));
4090 }
4091
4092
4093
4094 Ops[0] = Builder.CreateTrunc(Ops[0], OverloadedTy);
4095
4096
4097
4098
4099
4100
4101
4102
4105
4106
4107
4108 if (!TypeFlags.isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
4109 unsigned BytesPerElt =
4110 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
4111 Ops[3] = Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
4112 }
4113
4114 return Builder.CreateCall(F, Ops);
4115}
4116
4119 unsigned IntID) {
4120
4121
4122 auto *OverloadedTy = dyn_castllvm::ScalableVectorType(Ops[1]->getType());
4123 if (!OverloadedTy)
4125
4126
4128
4129
4130 if (Ops[1]->getType()->isVectorTy()) {
4131 if (Ops.size() == 3) {
4132
4133 Ops.push_back(ConstantInt::get(Int64Ty, 0));
4134
4135
4136 std::swap(Ops[2], Ops[3]);
4137 } else {
4138
4140 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
4141 if (BytesPerElt > 1)
4142 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
4143 }
4144 }
4145
4146 Function *F = CGM.getIntrinsic(IntID, OverloadedTy);
4147 return Builder.CreateCall(F, Ops);
4148}
4149
4152 unsigned IntID) {
4153 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
4155 Value *BasePtr = Ops[1];
4156
4157
4158 if (Ops.size() > 2)
4159 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]);
4160
4161 Function *F = CGM.getIntrinsic(IntID, {VTy});
4162 return Builder.CreateCall(F, {Predicate, BasePtr});
4163}
4164
4167 unsigned IntID) {
4168 llvm::ScalableVectorType *VTy = getSVEType(TypeFlags);
4169
4170 unsigned N;
4171 switch (IntID) {
4172 case Intrinsic::aarch64_sve_st2:
4173 case Intrinsic::aarch64_sve_st1_pn_x2:
4174 case Intrinsic::aarch64_sve_stnt1_pn_x2:
4175 case Intrinsic::aarch64_sve_st2q:
4176 N = 2;
4177 break;
4178 case Intrinsic::aarch64_sve_st3:
4179 case Intrinsic::aarch64_sve_st3q:
4180 N = 3;
4181 break;
4182 case Intrinsic::aarch64_sve_st4:
4183 case Intrinsic::aarch64_sve_st1_pn_x4:
4184 case Intrinsic::aarch64_sve_stnt1_pn_x4:
4185 case Intrinsic::aarch64_sve_st4q:
4186 N = 4;
4187 break;
4188 default:
4189 llvm_unreachable("unknown intrinsic!");
4190 }
4191
4193 Value *BasePtr = Ops[1];
4194
4195
4196 if (Ops.size() > (2 + N))
4197 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]);
4198
4199
4200
4202 for (unsigned I = Ops.size() - N; I < Ops.size(); ++I)
4203 Operands.push_back(Ops[I]);
4204 Operands.append({Predicate, BasePtr});
4205 Function *F = CGM.getIntrinsic(IntID, { VTy });
4206
4207 return Builder.CreateCall(F, Operands);
4208}
4209
4210
4211
4212
4215 unsigned BuiltinID) {
4216
4220 }
4221
4222
4225
4226
4227 llvm::ScalableVectorType *Ty = getSVEType(TypeFlags);
4229}
4230
4233 llvm::Type *OverloadedTy = getSVEType(TypeFlags);
4234 Function *F = CGM.getIntrinsic(BuiltinID, OverloadedTy);
4235 return Builder.CreateCall(F, {Ops[0], Builder.getInt32(0)});
4236}
4237
4240 unsigned BuiltinID) {
4243 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
4244
4246 Value *BasePtr = Ops[1];
4247
4248
4249 if (Ops.size() > 3)
4250 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
4251
4252 Value *PrfOp = Ops.back();
4253
4255 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
4256}
4257
4259 llvm::Type *ReturnTy,
4261 unsigned IntrinsicID,
4262 bool IsZExtReturn) {
4264 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
4266
4267
4268
4269 if (MemEltTy->isVectorTy()) {
4270 assert(MemEltTy == FixedVectorType::get(Int8Ty, 1) &&
4271 "Only <1 x i8> expected");
4273 }
4274
4275
4276
4278 llvm::ScalableVectorType *MemoryTy = nullptr;
4279 llvm::ScalableVectorType *PredTy = nullptr;
4280 bool IsQuadLoad = false;
4281 switch (IntrinsicID) {
4282 case Intrinsic::aarch64_sve_ld1uwq:
4283 case Intrinsic::aarch64_sve_ld1udq:
4284 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
4285 PredTy = llvm::ScalableVectorType::get(
4287 IsQuadLoad = true;
4288 break;
4289 default:
4290 MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
4291 PredTy = MemoryTy;
4292 break;
4293 }
4294
4296 Value *BasePtr = Ops[1];
4297
4298
4299 if (Ops.size() > 2)
4300 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
4301
4302 Function *F = CGM.getIntrinsic(IntrinsicID, IsQuadLoad ? VectorTy : MemoryTy);
4303 auto *Load =
4305 auto TBAAInfo = CGM.getTBAAAccessInfo(LangPTy->getPointeeType());
4306 CGM.DecorateInstructionWithTBAA(Load, TBAAInfo);
4307
4308 if (IsQuadLoad)
4309 return Load;
4310
4311 return IsZExtReturn ? Builder.CreateZExt(Load, VectorTy)
4312 : Builder.CreateSExt(Load, VectorTy);
4313}
4314
4317 unsigned IntrinsicID) {
4319 llvm::Type *MemEltTy = CGM.getTypes().ConvertType(
4321
4322
4323
4324 if (MemEltTy->isVectorTy()) {
4325 assert(MemEltTy == FixedVectorType::get(Int8Ty, 1) &&
4326 "Only <1 x i8> expected");
4328 }
4329
4330
4331
4333 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
4334
4335 auto PredTy = MemoryTy;
4336 auto AddrMemoryTy = MemoryTy;
4337 bool IsQuadStore = false;
4338
4339 switch (IntrinsicID) {
4340 case Intrinsic::aarch64_sve_st1wq:
4341 case Intrinsic::aarch64_sve_st1dq:
4342 AddrMemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
4343 PredTy =
4344 llvm::ScalableVectorType::get(IntegerType::get(getLLVMContext(), 1), 1);
4345 IsQuadStore = true;
4346 break;
4347 default:
4348 break;
4349 }
4351 Value *BasePtr = Ops[1];
4352
4353
4354 if (Ops.size() == 4)
4355 BasePtr = Builder.CreateGEP(AddrMemoryTy, BasePtr, Ops[2]);
4356
4357
4359 IsQuadStore ? Ops.back() : Builder.CreateTrunc(Ops.back(), MemoryTy);
4360
4362 CGM.getIntrinsic(IntrinsicID, IsQuadStore ? VectorTy : MemoryTy);
4363 auto *Store =
4365 auto TBAAInfo = CGM.getTBAAAccessInfo(LangPTy->getPointeeType());
4366 CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
4367 return Store;
4368}
4369
4372 unsigned IntID) {
4375
4377 NewOps.push_back(Ops[2]);
4378
4379 llvm::Value *BasePtr = Ops[3];
4380 llvm::Value *RealSlice = Ops[1];
4381
4382
4383 if (Ops.size() == 5) {
4384 Function *StreamingVectorLength =
4385 CGM.getIntrinsic(Intrinsic::aarch64_sme_cntsd);
4386 llvm::Value *StreamingVectorLengthCall =
4387 Builder.CreateMul(Builder.CreateCall(StreamingVectorLength),
4388 llvm::ConstantInt::get(Int64Ty, 8), "svl",
4389 true, true);
4390 llvm::Value *Mulvl =
4391 Builder.CreateMul(StreamingVectorLengthCall, Ops[4], "mulvl");
4392
4393 BasePtr = Builder.CreateGEP(Int8Ty, Ops[3], Mulvl);
4395 RealSlice = Builder.CreateAdd(RealSlice, Ops[4]);
4396 RealSlice = Builder.CreateTrunc(RealSlice, Int32Ty);
4397 }
4398 NewOps.push_back(BasePtr);
4399 NewOps.push_back(Ops[0]);
4400 NewOps.push_back(RealSlice);
4402 return Builder.CreateCall(F, NewOps);
4403}
4404
4407 unsigned IntID) {
4408 auto *VecTy = getSVEType(TypeFlags);
4409 Function *F = CGM.getIntrinsic(IntID, VecTy);
4414 return Builder.CreateCall(F, Ops);
4415}
4416
4419 unsigned IntID) {
4420
4421 if (Ops.size() == 0)
4422 Ops.push_back(llvm::ConstantInt::get(Int32Ty, 255));
4423 Function *F = CGM.getIntrinsic(IntID, {});
4424 return Builder.CreateCall(F, Ops);
4425}
4426
4429 unsigned IntID) {
4430 if (Ops.size() == 2)
4431 Ops.push_back(Builder.getInt32(0));
4432 else
4433 Ops[2] = Builder.CreateIntCast(Ops[2], Int32Ty, true);
4434 Function *F = CGM.getIntrinsic(IntID, {});
4435 return Builder.CreateCall(F, Ops);
4436}
4437
4438
4439
4441 return Builder.CreateVectorSplat(
4443}
4444
4446 if (auto *Ty = Scalar->getType(); Ty->isVectorTy()) {
4447#ifndef NDEBUG
4449 ElementCount EC = VecTy->getElementCount();
4450 assert(EC.isScalar() && VecTy->getElementType() == Int8Ty &&
4451 "Only <1 x i8> expected");
4452#endif
4453 Scalar = Builder.CreateExtractElement(Scalar, uint64_t(0));
4454 }
4456}
4457
4459
4460
4461
4462
4463
4464
4465
4466 if (auto *StructTy = dyn_cast(Ty)) {
4467 Value *Tuple = llvm::PoisonValue::get(Ty);
4468
4469 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
4470 Value *In = Builder.CreateExtractValue(Val, I);
4471 Value *Out = Builder.CreateBitCast(In, StructTy->getTypeAtIndex(I));
4472 Tuple = Builder.CreateInsertValue(Tuple, Out, I);
4473 }
4474
4475 return Tuple;
4476 }
4477
4478 return Builder.CreateBitCast(Val, Ty);
4479}
4480
4483 auto *SplatZero = Constant::getNullValue(Ty);
4484 Ops.insert(Ops.begin(), SplatZero);
4485}
4486
4489 auto *SplatUndef = UndefValue::get(Ty);
4490 Ops.insert(Ops.begin(), SplatUndef);
4491}
4492
4493SmallVector<llvm::Type *, 2>
4495 llvm::Type *ResultType,
4498 return {};
4499
4500 llvm::Type *DefaultType = getSVEType(TypeFlags);
4501
4503 return {DefaultType, Ops[1]->getType()};
4504
4506 return {getSVEPredType(TypeFlags), Ops[0]->getType()};
4507
4509 return {Ops[0]->getType(), Ops.back()->getType()};
4510
4511 if (TypeFlags.isReductionQV() && !ResultType->isScalableTy() &&
4512 ResultType->isVectorTy())
4513 return {ResultType, Ops[1]->getType()};
4514
4515 assert(TypeFlags.isOverloadDefault() && "Unexpected value for overloads");
4516 return {DefaultType};
4517}
4518
4522 "Expects TypleFlags.isTupleSet() or TypeFlags.isTupleGet()");
4524
4526 return Builder.CreateInsertValue(Ops[0], Ops[2], Idx);
4527 return Builder.CreateExtractValue(Ops[0], Idx);
4528}
4529
4531 llvm::Type *Ty,
4533 assert(TypeFlags.isTupleCreate() && "Expects TypleFlag isTupleCreate");
4534
4535 Value *Tuple = llvm::PoisonValue::get(Ty);
4536 for (unsigned Idx = 0; Idx < Ops.size(); Idx++)
4537 Tuple = Builder.CreateInsertValue(Tuple, Ops[Idx], Idx);
4538
4539 return Tuple;
4540}
4541
4545
4546 unsigned ICEArguments = 0;
4550
4551
4552
4554
4555 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
4556 bool IsICE = ICEArguments & (1 << i);
4558
4559 if (IsICE) {
4560
4561
4562 std::optionalllvm::APSInt Result =
4564 assert(Result && "Expected argument to be a constant");
4565
4566
4567
4568
4571 continue;
4572 }
4573
4574 if (isa(Arg->getType()) && !IsTupleGetOrSet) {
4575 for (unsigned I = 0; I < Arg->getType()->getStructNumElements(); ++I)
4576 Ops.push_back(Builder.CreateExtractValue(Arg, I));
4577
4578 continue;
4579 }
4580
4581 Ops.push_back(Arg);
4582 }
4583}
4584
4588 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
4589 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) {
4592 }
4593
4596
4600
4601 if (TypeFlags.isLoad())
4604 else if (TypeFlags.isStore())
4622 else if (TypeFlags.isUndef())
4623 return UndefValue::get(Ty);
4624 else if (Builtin->LLVMIntrinsic != 0) {
4625
4627 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_set_fpmr),
4628 Ops.pop_back_val());
4629 if (TypeFlags.getMergeType() == SVETypeFlags::MergeZeroExp)
4631
4632 if (TypeFlags.getMergeType() == SVETypeFlags::MergeAnyExp)
4634
4635
4636
4638 Ops.push_back(Builder.getInt32( 31));
4640 Ops.insert(&Ops[1], Builder.getInt32( 31));
4641
4642
4643 for (Value *&Op : Ops)
4644 if (auto PredTy = dyn_castllvm::VectorType(Op->getType()))
4645 if (PredTy->getElementType()->isIntegerTy(1))
4647
4648
4652 }
4653
4655 std::swap(Ops[1], Ops[2]);
4657 std::swap(Ops[1], Ops[2]);
4659 TypeFlags.getMergeType() == SVETypeFlags::MergeAny)
4660 std::swap(Ops[1], Ops[2]);
4662 TypeFlags.getMergeType() == SVETypeFlags::MergeAny)
4663 std::swap(Ops[1], Ops[3]);
4664
4665
4666 if (TypeFlags.getMergeType() == SVETypeFlags::MergeZero) {
4667 llvm::Type *OpndTy = Ops[1]->getType();
4668 auto *SplatZero = Constant::getNullValue(OpndTy);
4669 Ops[1] = Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
4670 }
4671
4675
4676 if (Call->getType() == Ty)
4677 return Call;
4678
4679
4680 if (auto PredTy = dyn_castllvm::ScalableVectorType(Ty))
4682 if (auto PredTupleTy = dyn_castllvm::StructType(Ty))
4684
4685 llvm_unreachable("unsupported element count!");
4686 }
4687
4688 switch (BuiltinID) {
4689 default:
4690 return nullptr;
4691
4692 case SVE::BI__builtin_sve_svreinterpret_b: {
4693 auto SVCountTy =
4694 llvm::TargetExtType::get(getLLVMContext(), "aarch64.svcount");
4695 Function *CastFromSVCountF =
4696 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
4697 return Builder.CreateCall(CastFromSVCountF, Ops[0]);
4698 }
4699 case SVE::BI__builtin_sve_svreinterpret_c: {
4700 auto SVCountTy =
4701 llvm::TargetExtType::get(getLLVMContext(), "aarch64.svcount");
4703 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
4704 return Builder.CreateCall(CastToSVCountF, Ops[0]);
4705 }
4706
4707 case SVE::BI__builtin_sve_svpsel_lane_b8:
4708 case SVE::BI__builtin_sve_svpsel_lane_b16:
4709 case SVE::BI__builtin_sve_svpsel_lane_b32:
4710 case SVE::BI__builtin_sve_svpsel_lane_b64:
4711 case SVE::BI__builtin_sve_svpsel_lane_c8:
4712 case SVE::BI__builtin_sve_svpsel_lane_c16:
4713 case SVE::BI__builtin_sve_svpsel_lane_c32:
4714 case SVE::BI__builtin_sve_svpsel_lane_c64: {
4717 "aarch64.svcount")) &&
4718 "Unexpected TargetExtType");
4719 auto SVCountTy =
4720 llvm::TargetExtType::get(getLLVMContext(), "aarch64.svcount");
4721 Function *CastFromSVCountF =
4722 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
4724 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
4725
4727 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_psel, OverloadedTy);
4728 llvm::Value *Ops0 =
4729 IsSVCount ? Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0];
4731 llvm::Value *PSel = Builder.CreateCall(F, {Ops0, Ops1, Ops[2]});
4732 return IsSVCount ? Builder.CreateCall(CastToSVCountF, PSel) : PSel;
4733 }
4734 case SVE::BI__builtin_sve_svmov_b_z: {
4735
4737 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
4738 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_and_z, OverloadedTy);
4739 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
4740 }
4741
4742 case SVE::BI__builtin_sve_svnot_b_z: {
4743
4745 llvm::Type* OverloadedTy = getSVEType(TypeFlags);
4746 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_eor_z, OverloadedTy);
4747 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
4748 }
4749
4750 case SVE::BI__builtin_sve_svmovlb_u16:
4751 case SVE::BI__builtin_sve_svmovlb_u32:
4752 case SVE::BI__builtin_sve_svmovlb_u64:
4753 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
4754
4755 case SVE::BI__builtin_sve_svmovlb_s16:
4756 case SVE::BI__builtin_sve_svmovlb_s32:
4757 case SVE::BI__builtin_sve_svmovlb_s64:
4758 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
4759
4760 case SVE::BI__builtin_sve_svmovlt_u16:
4761 case SVE::BI__builtin_sve_svmovlt_u32:
4762 case SVE::BI__builtin_sve_svmovlt_u64:
4763 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
4764
4765 case SVE::BI__builtin_sve_svmovlt_s16:
4766 case SVE::BI__builtin_sve_svmovlt_s32:
4767 case SVE::BI__builtin_sve_svmovlt_s64:
4768 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
4769
4770 case SVE::BI__builtin_sve_svpmullt_u16:
4771 case SVE::BI__builtin_sve_svpmullt_u64:
4772 case SVE::BI__builtin_sve_svpmullt_n_u16:
4773 case SVE::BI__builtin_sve_svpmullt_n_u64:
4774 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
4775
4776 case SVE::BI__builtin_sve_svpmullb_u16:
4777 case SVE::BI__builtin_sve_svpmullb_u64:
4778 case SVE::BI__builtin_sve_svpmullb_n_u16:
4779 case SVE::BI__builtin_sve_svpmullb_n_u64:
4780 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
4781
4782 case SVE::BI__builtin_sve_svdup_n_b8:
4783 case SVE::BI__builtin_sve_svdup_n_b16:
4784 case SVE::BI__builtin_sve_svdup_n_b32:
4785 case SVE::BI__builtin_sve_svdup_n_b64: {
4787 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType()));
4788 llvm::ScalableVectorType *OverloadedTy = getSVEType(TypeFlags);
4791 }
4792
4793 case SVE::BI__builtin_sve_svdupq_n_b8:
4794 case SVE::BI__builtin_sve_svdupq_n_b16:
4795 case SVE::BI__builtin_sve_svdupq_n_b32:
4796 case SVE::BI__builtin_sve_svdupq_n_b64:
4797 case SVE::BI__builtin_sve_svdupq_n_u8:
4798 case SVE::BI__builtin_sve_svdupq_n_s8:
4799 case SVE::BI__builtin_sve_svdupq_n_u64:
4800 case SVE::BI__builtin_sve_svdupq_n_f64:
4801 case SVE::BI__builtin_sve_svdupq_n_s64:
4802 case SVE::BI__builtin_sve_svdupq_n_u16:
4803 case SVE::BI__builtin_sve_svdupq_n_f16:
4804 case SVE::BI__builtin_sve_svdupq_n_bf16:
4805 case SVE::BI__builtin_sve_svdupq_n_s16:
4806 case SVE::BI__builtin_sve_svdupq_n_u32:
4807 case SVE::BI__builtin_sve_svdupq_n_f32:
4808 case SVE::BI__builtin_sve_svdupq_n_s32: {
4809
4810
4811 unsigned NumOpnds = Ops.size();
4812
4813 bool IsBoolTy =
4815
4816
4817
4818
4819 llvm::Type *EltTy = Ops[0]->getType();
4820 if (IsBoolTy)
4822
4824 for (unsigned I = 0; I < NumOpnds; ++I)
4825 VecOps.push_back(Builder.CreateZExt(Ops[I], EltTy));
4827
4829 Value *InsertSubVec = Builder.CreateInsertVector(
4830 OverloadedTy, PoisonValue::get(OverloadedTy), Vec, uint64_t(0));
4831
4833 CGM.getIntrinsic(Intrinsic::aarch64_sve_dupq_lane, OverloadedTy);
4834 Value *DupQLane =
4835 Builder.CreateCall(F, {InsertSubVec, Builder.getInt64(0)});
4836
4837 if (!IsBoolTy)
4838 return DupQLane;
4839
4842
4843
4844 F = CGM.getIntrinsic(NumOpnds == 2 ? Intrinsic::aarch64_sve_cmpne
4845 : Intrinsic::aarch64_sve_cmpne_wide,
4846 OverloadedTy);
4850 }
4851
4852 case SVE::BI__builtin_sve_svpfalse_b:
4853 return ConstantInt::getFalse(Ty);
4854
4855 case SVE::BI__builtin_sve_svpfalse_c: {
4856 auto SVBoolTy = ScalableVectorType::get(Builder.getInt1Ty(), 16);
4858 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, Ty);
4859 return Builder.CreateCall(CastToSVCountF, ConstantInt::getFalse(SVBoolTy));
4860 }
4861
4862 case SVE::BI__builtin_sve_svlen_bf16:
4863 case SVE::BI__builtin_sve_svlen_f16:
4864 case SVE::BI__builtin_sve_svlen_f32:
4865 case SVE::BI__builtin_sve_svlen_f64:
4866 case SVE::BI__builtin_sve_svlen_s8:
4867 case SVE::BI__builtin_sve_svlen_s16:
4868 case SVE::BI__builtin_sve_svlen_s32:
4869 case SVE::BI__builtin_sve_svlen_s64:
4870 case SVE::BI__builtin_sve_svlen_u8:
4871 case SVE::BI__builtin_sve_svlen_u16:
4872 case SVE::BI__builtin_sve_svlen_u32:
4873 case SVE::BI__builtin_sve_svlen_u64: {
4875 return Builder.CreateElementCount(Ty, getSVEType(TF)->getElementCount());
4876 }
4877
4878 case SVE::BI__builtin_sve_svtbl2_u8:
4879 case SVE::BI__builtin_sve_svtbl2_s8:
4880 case SVE::BI__builtin_sve_svtbl2_u16:
4881 case SVE::BI__builtin_sve_svtbl2_s16:
4882 case SVE::BI__builtin_sve_svtbl2_u32:
4883 case SVE::BI__builtin_sve_svtbl2_s32:
4884 case SVE::BI__builtin_sve_svtbl2_u64:
4885 case SVE::BI__builtin_sve_svtbl2_s64:
4886 case SVE::BI__builtin_sve_svtbl2_f16:
4887 case SVE::BI__builtin_sve_svtbl2_bf16:
4888 case SVE::BI__builtin_sve_svtbl2_f32:
4889 case SVE::BI__builtin_sve_svtbl2_f64: {
4892 return Builder.CreateCall(F, Ops);
4893 }
4894
4895 case SVE::BI__builtin_sve_svset_neonq_s8:
4896 case SVE::BI__builtin_sve_svset_neonq_s16:
4897 case SVE::BI__builtin_sve_svset_neonq_s32:
4898 case SVE::BI__builtin_sve_svset_neonq_s64:
4899 case SVE::BI__builtin_sve_svset_neonq_u8:
4900 case SVE::BI__builtin_sve_svset_neonq_u16:
4901 case SVE::BI__builtin_sve_svset_neonq_u32:
4902 case SVE::BI__builtin_sve_svset_neonq_u64:
4903 case SVE::BI__builtin_sve_svset_neonq_f16:
4904 case SVE::BI__builtin_sve_svset_neonq_f32:
4905 case SVE::BI__builtin_sve_svset_neonq_f64:
4906 case SVE::BI__builtin_sve_svset_neonq_bf16: {
4907 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], uint64_t(0));
4908 }
4909
4910 case SVE::BI__builtin_sve_svget_neonq_s8:
4911 case SVE::BI__builtin_sve_svget_neonq_s16:
4912 case SVE::BI__builtin_sve_svget_neonq_s32:
4913 case SVE::BI__builtin_sve_svget_neonq_s64:
4914 case SVE::BI__builtin_sve_svget_neonq_u8:
4915 case SVE::BI__builtin_sve_svget_neonq_u16:
4916 case SVE::BI__builtin_sve_svget_neonq_u32:
4917 case SVE::BI__builtin_sve_svget_neonq_u64:
4918 case SVE::BI__builtin_sve_svget_neonq_f16:
4919 case SVE::BI__builtin_sve_svget_neonq_f32:
4920 case SVE::BI__builtin_sve_svget_neonq_f64:
4921 case SVE::BI__builtin_sve_svget_neonq_bf16: {
4922 return Builder.CreateExtractVector(Ty, Ops[0], uint64_t(0));
4923 }
4924
4925 case SVE::BI__builtin_sve_svdup_neonq_s8:
4926 case SVE::BI__builtin_sve_svdup_neonq_s16:
4927 case SVE::BI__builtin_sve_svdup_neonq_s32:
4928 case SVE::BI__builtin_sve_svdup_neonq_s64:
4929 case SVE::BI__builtin_sve_svdup_neonq_u8:
4930 case SVE::BI__builtin_sve_svdup_neonq_u16:
4931 case SVE::BI__builtin_sve_svdup_neonq_u32:
4932 case SVE::BI__builtin_sve_svdup_neonq_u64:
4933 case SVE::BI__builtin_sve_svdup_neonq_f16:
4934 case SVE::BI__builtin_sve_svdup_neonq_f32:
4935 case SVE::BI__builtin_sve_svdup_neonq_f64:
4936 case SVE::BI__builtin_sve_svdup_neonq_bf16: {
4937 Value *Insert = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
4938 uint64_t(0));
4939 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
4940 {Insert, Builder.getInt64(0)});
4941 }
4942 }
4943
4944
4945 return nullptr;
4946}
4947
4950 unsigned MultiVec;
4951 switch (BuiltinID) {
4952 default:
4953 return;
4954 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1:
4955 MultiVec = 1;
4956 break;
4957 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2:
4958 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2:
4959 MultiVec = 2;
4960 break;
4961 case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4:
4962 case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4:
4963 MultiVec = 4;
4964 break;
4965 }
4966
4967 if (MultiVec > 0)
4968 for (unsigned I = 0; I < MultiVec; ++I)
4969 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]);
4970}
4971
4976
4980
4985 else if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
4986 BuiltinID == SME::BI__builtin_sme_svzero_za)
4988 else if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
4989 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za ||
4990 BuiltinID == SME::BI__builtin_sme_svldr_za ||
4991 BuiltinID == SME::BI__builtin_sme_svstr_za)
4993
4994
4996 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_set_fpmr),
4997 Ops.pop_back_val());
4998
5000
5001 auto isCntsBuiltin = [&]() {
5002 switch (BuiltinID) {
5003 default:
5004 return 0;
5005 case SME::BI__builtin_sme_svcntsb:
5006 return 8;
5007 case SME::BI__builtin_sme_svcntsh:
5008 return 4;
5009 case SME::BI__builtin_sme_svcntsw:
5010 return 2;
5011 }
5012 };
5013
5014 if (auto Mul = isCntsBuiltin()) {
5015 llvm::Value *Cntd =
5016 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_sme_cntsd));
5017 return Builder.CreateMul(Cntd, llvm::ConstantInt::get(Int64Ty, Mul),
5018 "mulsvl", true, true);
5019 }
5020
5021
5022 if (Builtin->LLVMIntrinsic == 0)
5023 return nullptr;
5024
5025
5026 for (Value *&Op : Ops)
5027 if (auto PredTy = dyn_castllvm::VectorType(Op->getType()))
5028 if (PredTy->getElementType()->isIntegerTy(1))
5030
5033 ? CGM.getIntrinsic(Builtin->LLVMIntrinsic)
5034 : CGM.getIntrinsic(Builtin->LLVMIntrinsic, {getSVEType(TypeFlags)});
5035
5036 return Builder.CreateCall(F, Ops);
5037}
5038
5039
5040
5043 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")};
5044 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
5045 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
5046 llvm::Function *F =
5048 llvm::Value *X18 = CGF.Builder.CreateCall(F, Metadata);
5050}
5051
5054 llvm::Triple::ArchType Arch) {
5058
5062
5063 if (BuiltinID == Builtin::BI__builtin_cpu_supports)
5064 return EmitAArch64CpuSupports(E);
5065
5066 unsigned HintID = static_cast<unsigned>(-1);
5067 switch (BuiltinID) {
5068 default: break;
5069 case clang::AArch64::BI__builtin_arm_nop:
5070 HintID = 0;
5071 break;
5072 case clang::AArch64::BI__builtin_arm_yield:
5073 case clang::AArch64::BI__yield:
5074 HintID = 1;
5075 break;
5076 case clang::AArch64::BI__builtin_arm_wfe:
5077 case clang::AArch64::BI__wfe:
5078 HintID = 2;
5079 break;
5080 case clang::AArch64::BI__builtin_arm_wfi:
5081 case clang::AArch64::BI__wfi:
5082 HintID = 3;
5083 break;
5084 case clang::AArch64::BI__builtin_arm_sev:
5085 case clang::AArch64::BI__sev:
5086 HintID = 4;
5087 break;
5088 case clang::AArch64::BI__builtin_arm_sevl:
5089 case clang::AArch64::BI__sevl:
5090 HintID = 5;
5091 break;
5092 }
5093
5094 if (HintID != static_cast<unsigned>(-1)) {
5095 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint);
5096 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
5097 }
5098
5099 if (BuiltinID == clang::AArch64::BI__builtin_arm_trap) {
5100 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_break);
5102 return Builder.CreateCall(F, Builder.CreateZExt(Arg, CGM.Int32Ty));
5103 }
5104
5105 if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
5106
5108 llvm::FunctionType::get(StructType::get(CGM.Int64Ty, CGM.Int64Ty), {},
5109 false),
5110 "__arm_sme_state"));
5111 auto Attrs = AttributeList().addFnAttribute(getLLVMContext(),
5112 "aarch64_pstate_sm_compatible");
5113 CI->setAttributes(Attrs);
5114 CI->setCallingConv(
5115 llvm::CallingConv::
5116 AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
5117 Builder.CreateStore(Builder.CreateExtractValue(CI, 0),
5119 return Builder.CreateStore(Builder.CreateExtractValue(CI, 1),
5121 }
5122
5123 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
5125 "rbit of unusual size!");
5127 return Builder.CreateCall(
5128 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
5129 }
5130 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
5132 "rbit of unusual size!");
5134 return Builder.CreateCall(
5135 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
5136 }
5137
5138 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
5139 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
5141 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
5143 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
5144 Res = Builder.CreateTrunc(Res, Builder.getInt32Ty());
5145 return Res;
5146 }
5147
5148 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
5150 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls), Arg,
5151 "cls");
5152 }
5153 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
5155 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls64), Arg,
5156 "cls");
5157 }
5158
5159 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
5160 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
5162 llvm::Type *Ty = Arg->getType();
5163 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32z, Ty),
5164 Arg, "frint32z");
5165 }
5166
5167 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
5168 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
5170 llvm::Type *Ty = Arg->getType();
5171 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64z, Ty),
5172 Arg, "frint64z");
5173 }
5174
5175 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
5176 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
5178 llvm::Type *Ty = Arg->getType();
5179 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32x, Ty),
5180 Arg, "frint32x");
5181 }
5182
5183 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
5184 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
5186 llvm::Type *Ty = Arg->getType();
5187 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64x, Ty),
5188 Arg, "frint64x");
5189 }
5190
5191 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
5193 "__jcvt of unusual size!");
5195 return Builder.CreateCall(
5196 CGM.getIntrinsic(Intrinsic::aarch64_fjcvtzs), Arg);
5197 }
5198
5199 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
5200 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
5201 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
5202 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
5205
5206 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
5207
5208
5209 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_ld64b);
5210 llvm::Value *Val = Builder.CreateCall(F, MemAddr);
5211 llvm::Value *ToRet;
5212 for (size_t i = 0; i < 8; i++) {
5213 llvm::Value *ValOffsetPtr =
5217 ToRet = Builder.CreateStore(Builder.CreateExtractValue(Val, i), Addr);
5218 }
5219 return ToRet;
5220 } else {
5221
5222
5224 Args.push_back(MemAddr);
5225 for (size_t i = 0; i < 8; i++) {
5226 llvm::Value *ValOffsetPtr =
5230 Args.push_back(Builder.CreateLoad(Addr));
5231 }
5232
5233 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
5234 ? Intrinsic::aarch64_st64b
5235 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
5236 ? Intrinsic::aarch64_st64bv
5237 : Intrinsic::aarch64_st64bv0);
5239 return Builder.CreateCall(F, Args);
5240 }
5241 }
5242
5243 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
5244 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
5245
5246 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
5247 ? Intrinsic::aarch64_rndr
5248 : Intrinsic::aarch64_rndrrs);
5250 llvm::Value *Val = Builder.CreateCall(F);
5251 Value *RandomValue = Builder.CreateExtractValue(Val, 0);
5252 Value *Status = Builder.CreateExtractValue(Val, 1);
5253
5255 Builder.CreateStore(RandomValue, MemAddress);
5257 return Status;
5258 }
5259
5260 if (BuiltinID == clang::AArch64::BI__clear_cache) {
5261 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
5264 for (unsigned i = 0; i < 2; i++)
5266 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
5268 StringRef Name = FD->getName();
5270 }
5271
5272 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
5273 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
5276 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
5277 ? Intrinsic::aarch64_ldaxp
5278 : Intrinsic::aarch64_ldxp);
5279
5281 Value *Val = Builder.CreateCall(F, LdPtr, "ldxp");
5282
5283 Value *Val0 = Builder.CreateExtractValue(Val, 1);
5284 Value *Val1 = Builder.CreateExtractValue(Val, 0);
5285 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
5286 Val0 = Builder.CreateZExt(Val0, Int128Ty);
5287 Val1 = Builder.CreateZExt(Val1, Int128Ty);
5288
5289 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
5290 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true );
5291 Val = Builder.CreateOr(Val, Val1);
5293 } else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
5294 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
5296
5298 llvm::Type *RealResTy = ConvertType(Ty);
5299 llvm::Type *IntTy =
5301
5303 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_ldaex
5304 ? Intrinsic::aarch64_ldaxr
5305 : Intrinsic::aarch64_ldxr,
5307 CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
5308 Val->addParamAttr(
5310
5311 if (RealResTy->isPointerTy())
5312 return Builder.CreateIntToPtr(Val, RealResTy);
5313
5314 llvm::Type *IntResTy = llvm::IntegerType::get(
5315 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
5316 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
5317 RealResTy);
5318 }
5319
5320 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
5321 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
5324 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_stlex
5325 ? Intrinsic::aarch64_stlxp
5326 : Intrinsic::aarch64_stxp);
5327 llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty);
5328
5331
5333 llvm::Value *Val = Builder.CreateLoad(Tmp);
5334
5335 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
5336 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
5338 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp");
5339 }
5340
5341 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
5342 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
5345
5347 llvm::Type *StoreTy =
5349
5350 if (StoreVal->getType()->isPointerTy())
5351 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty);
5352 else {
5353 llvm::Type *IntTy = llvm::IntegerType::get(
5355 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
5356 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
5357 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
5358 }
5359
5361 CGM.getIntrinsic(BuiltinID == clang::AArch64::BI__builtin_arm_stlex
5362 ? Intrinsic::aarch64_stlxr
5363 : Intrinsic::aarch64_stxr,
5365 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
5366 CI->addParamAttr(
5367 1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
5368 return CI;
5369 }
5370
5371 if (BuiltinID == clang::AArch64::BI__getReg) {
5374 llvm_unreachable("Sema will ensure that the parameter is constant");
5375
5376 llvm::APSInt Value = Result.Val.getInt();
5377 LLVMContext &Context = CGM.getLLVMContext();
5379
5380 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
5381 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
5382 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
5383
5384 llvm::Function *F =
5385 CGM.getIntrinsic(Intrinsic::read_register, {Int64Ty});
5386 return Builder.CreateCall(F, Metadata);
5387 }
5388
5389 if (BuiltinID == clang::AArch64::BI__break) {
5392 llvm_unreachable("Sema will ensure that the parameter is constant");
5393
5394 llvm::Function *F = CGM.getIntrinsic(Intrinsic::aarch64_break);
5396 }
5397
5398 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
5399 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
5400 return Builder.CreateCall(F);
5401 }
5402
5403 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
5404 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
5405 llvm::SyncScope::SingleThread);
5406
5407
5408 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
5409 switch (BuiltinID) {
5410 case clang::AArch64::BI__builtin_arm_crc32b:
5411 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
5412 case clang::AArch64::BI__builtin_arm_crc32cb:
5413 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
5414 case clang::AArch64::BI__builtin_arm_crc32h:
5415 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
5416 case clang::AArch64::BI__builtin_arm_crc32ch:
5417 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
5418 case clang::AArch64::BI__builtin_arm_crc32w:
5419 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
5420 case clang::AArch64::BI__builtin_arm_crc32cw:
5421 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
5422 case clang::AArch64::BI__builtin_arm_crc32d:
5423 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
5424 case clang::AArch64::BI__builtin_arm_crc32cd:
5425 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
5426 }
5427
5428 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
5431 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
5432
5433 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
5434 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
5435
5436 return Builder.CreateCall(F, {Arg0, Arg1});
5437 }
5438
5439
5440 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
5446 return Builder.CreateCall(
5447 CGM.getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
5448 }
5449
5450
5451 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
5452 switch (BuiltinID) {
5453 case clang::AArch64::BI__builtin_arm_irg:
5454 MTEIntrinsicID = Intrinsic::aarch64_irg; break;
5455 case clang::AArch64::BI__builtin_arm_addg:
5456 MTEIntrinsicID = Intrinsic::aarch64_addg; break;
5457 case clang::AArch64::BI__builtin_arm_gmi:
5458 MTEIntrinsicID = Intrinsic::aarch64_gmi; break;
5459 case clang::AArch64::BI__builtin_arm_ldg:
5460 MTEIntrinsicID = Intrinsic::aarch64_ldg; break;
5461 case clang::AArch64::BI__builtin_arm_stg:
5462 MTEIntrinsicID = Intrinsic::aarch64_stg; break;
5463 case clang::AArch64::BI__builtin_arm_subp:
5464 MTEIntrinsicID = Intrinsic::aarch64_subp; break;
5465 }
5466
5467 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
5468 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
5471
5473 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
5474 {Pointer, Mask});
5475 }
5476 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
5479
5481 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
5482 {Pointer, TagOffset});
5483 }
5484 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
5487
5488 ExcludedMask = Builder.CreateZExt(ExcludedMask, Int64Ty);
5489 return Builder.CreateCall(
5490 CGM.getIntrinsic(MTEIntrinsicID), {Pointer, ExcludedMask});
5491 }
5492
5493
5494
5495 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
5497 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
5498 {TagAddress, TagAddress});
5499 }
5500
5501
5502
5503 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
5505 return Builder.CreateCall(CGM.getIntrinsic(MTEIntrinsicID),
5506 {TagAddress, TagAddress});
5507 }
5508 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
5511 return Builder.CreateCall(
5512 CGM.getIntrinsic(MTEIntrinsicID), {PointerA, PointerB});
5513 }
5514 }
5515
5516 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5517 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
5518 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5519 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
5520 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
5521 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
5522 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
5523 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
5524
5526 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5527 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
5528 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5529 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
5531
5532 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
5533 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
5534
5535 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
5536 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
5537
5538 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
5539 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
5540
5541 llvm::Type *ValueType;
5543 if (Is32Bit) {
5545 } else if (Is128Bit) {
5546 llvm::Type *Int128Ty =
5547 llvm::IntegerType::getInt128Ty(CGM.getLLVMContext());
5548 ValueType = Int128Ty;
5550 } else if (IsPointerBuiltin) {
5552 } else {
5554 };
5555
5557 AccessKind);
5558 }
5559
5560 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
5561 BuiltinID == clang::AArch64::BI_WriteStatusReg ||
5562 BuiltinID == clang::AArch64::BI__sys) {
5563 LLVMContext &Context = CGM.getLLVMContext();
5564
5565 unsigned SysReg =
5567
5568 std::string SysRegStr;
5569 unsigned SysRegOp0 = (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
5570 BuiltinID == clang::AArch64::BI_WriteStatusReg)
5571 ? ((1 << 1) | ((SysReg >> 14) & 1))
5572 : 1;
5573 llvm::raw_string_ostream(SysRegStr)
5574 << SysRegOp0 << ":" << ((SysReg >> 11) & 7) << ":"
5575 << ((SysReg >> 7) & 15) << ":" << ((SysReg >> 3) & 15) << ":"
5576 << (SysReg & 7);
5577
5578 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
5579 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
5580 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
5581
5584
5585 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
5586 llvm::Function *F = CGM.getIntrinsic(Intrinsic::read_register, Types);
5587
5588 return Builder.CreateCall(F, Metadata);
5589 }
5590
5591 llvm::Function *F = CGM.getIntrinsic(Intrinsic::write_register, Types);
5593 llvm::Value *Result = Builder.CreateCall(F, {Metadata, ArgValue});
5594 if (BuiltinID == clang::AArch64::BI__sys) {
5595
5596
5597 Result = ConstantInt::get(Builder.getInt32Ty(), 0);
5598 }
5600 }
5601
5602 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
5603 llvm::Function *F =
5604 CGM.getIntrinsic(Intrinsic::addressofreturnaddress, AllocaInt8PtrTy);
5605 return Builder.CreateCall(F);
5606 }
5607
5608 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
5609 llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
5610 return Builder.CreateCall(F);
5611 }
5612
5613 if (BuiltinID == clang::AArch64::BI__mulh ||
5614 BuiltinID == clang::AArch64::BI__umulh) {
5616 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
5617
5618 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
5623
5624 Value *MulResult, *HigherBits;
5625 if (IsSigned) {
5626 MulResult = Builder.CreateNSWMul(LHS, RHS);
5627 HigherBits = Builder.CreateAShr(MulResult, 64);
5628 } else {
5629 MulResult = Builder.CreateNUWMul(LHS, RHS);
5630 HigherBits = Builder.CreateLShr(MulResult, 64);
5631 }
5632 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
5633
5634 return HigherBits;
5635 }
5636
5637 if (BuiltinID == AArch64::BI__writex18byte ||
5638 BuiltinID == AArch64::BI__writex18word ||
5639 BuiltinID == AArch64::BI__writex18dword ||
5640 BuiltinID == AArch64::BI__writex18qword) {
5641
5644
5645
5647
5648
5651 StoreInst *Store =
5653 return Store;
5654 }
5655
5656 if (BuiltinID == AArch64::BI__readx18byte ||
5657 BuiltinID == AArch64::BI__readx18word ||
5658 BuiltinID == AArch64::BI__readx18dword ||
5659 BuiltinID == AArch64::BI__readx18qword) {
5660
5662
5663
5665
5666
5671 return Load;
5672 }
5673
5674 if (BuiltinID == AArch64::BI__addx18byte ||
5675 BuiltinID == AArch64::BI__addx18word ||
5676 BuiltinID == AArch64::BI__addx18dword ||
5677 BuiltinID == AArch64::BI__addx18qword ||
5678 BuiltinID == AArch64::BI__incx18byte ||
5679 BuiltinID == AArch64::BI__incx18word ||
5680 BuiltinID == AArch64::BI__incx18dword ||
5681 BuiltinID == AArch64::BI__incx18qword) {
5682 llvm::Type *IntTy;
5683 bool isIncrement;
5684 switch (BuiltinID) {
5685 case AArch64::BI__incx18byte:
5687 isIncrement = true;
5688 break;
5689 case AArch64::BI__incx18word:
5691 isIncrement = true;
5692 break;
5693 case AArch64::BI__incx18dword:
5695 isIncrement = true;
5696 break;
5697 case AArch64::BI__incx18qword:
5699 isIncrement = true;
5700 break;
5701 default:
5703 isIncrement = false;
5704 break;
5705 }
5706
5708 Value *ValToAdd =
5710
5711
5713
5714
5718
5719
5720 Value *AddResult = Builder.CreateAdd(Load, ValToAdd);
5721
5722
5723 StoreInst *Store =
5725 return Store;
5726 }
5727
5728 if (BuiltinID == AArch64::BI_CopyDoubleFromInt64 ||
5729 BuiltinID == AArch64::BI_CopyFloatFromInt32 ||
5730 BuiltinID == AArch64::BI_CopyInt32FromFloat ||
5731 BuiltinID == AArch64::BI_CopyInt64FromDouble) {
5734 return Builder.CreateBitCast(Arg, RetTy);
5735 }
5736
5737 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5738 BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5739 BuiltinID == AArch64::BI_CountLeadingZeros ||
5740 BuiltinID == AArch64::BI_CountLeadingZeros64) {
5743
5744 if (BuiltinID == AArch64::BI_CountLeadingOnes ||
5745 BuiltinID == AArch64::BI_CountLeadingOnes64)
5746 Arg = Builder.CreateXor(Arg, Constant::getAllOnesValue(ArgType));
5747
5750
5751 if (BuiltinID == AArch64::BI_CountLeadingOnes64 ||
5752 BuiltinID == AArch64::BI_CountLeadingZeros64)
5755 }
5756
5757 if (BuiltinID == AArch64::BI_CountLeadingSigns ||
5758 BuiltinID == AArch64::BI_CountLeadingSigns64) {
5760
5761 Function *F = (BuiltinID == AArch64::BI_CountLeadingSigns)
5762 ? CGM.getIntrinsic(Intrinsic::aarch64_cls)
5763 : CGM.getIntrinsic(Intrinsic::aarch64_cls64);
5764
5766 if (BuiltinID == AArch64::BI_CountLeadingSigns64)
5769 }
5770
5771 if (BuiltinID == AArch64::BI_CountOneBits ||
5772 BuiltinID == AArch64::BI_CountOneBits64) {
5776
5778 if (BuiltinID == AArch64::BI_CountOneBits64)
5781 }
5782
5783 if (BuiltinID == AArch64::BI__prefetch) {
5785 Value *RW = llvm::ConstantInt::get(Int32Ty, 0);
5786 Value *Locality = ConstantInt::get(Int32Ty, 3);
5790 }
5791
5792 if (BuiltinID == AArch64::BI__hlt) {
5793 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hlt);
5795
5796
5797
5798 return ConstantInt::get(Builder.getInt32Ty(), 0);
5799 }
5800
5801 if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
5802 return Builder.CreateFPTrunc(
5806
5807
5808
5809 if (std::optional MsvcIntId =
5812
5813
5815 return P.first == BuiltinID;
5816 });
5818 BuiltinID = It->second;
5819
5820
5821
5822 unsigned ICEArguments = 0;
5826
5829 for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) {
5830 if (i == 0) {
5831 switch (BuiltinID) {
5832 case NEON::BI__builtin_neon_vld1_v:
5833 case NEON::BI__builtin_neon_vld1q_v:
5834 case NEON::BI__builtin_neon_vld1_dup_v:
5835 case NEON::BI__builtin_neon_vld1q_dup_v:
5836 case NEON::BI__builtin_neon_vld1_lane_v:
5837 case NEON::BI__builtin_neon_vld1q_lane_v:
5838 case NEON::BI__builtin_neon_vst1_v:
5839 case NEON::BI__builtin_neon_vst1q_v:
5840 case NEON::BI__builtin_neon_vst1_lane_v:
5841 case NEON::BI__builtin_neon_vst1q_lane_v:
5842 case NEON::BI__builtin_neon_vldap1_lane_s64:
5843 case NEON::BI__builtin_neon_vldap1q_lane_s64:
5844 case NEON::BI__builtin_neon_vstl1_lane_s64:
5845 case NEON::BI__builtin_neon_vstl1q_lane_s64:
5846
5847
5850 continue;
5851 }
5852 }
5854 }
5855
5859
5863 assert(Result && "SISD intrinsic should have been handled");
5865 }
5866
5869 if (std::optionalllvm::APSInt Result =
5871
5873
5874 bool usgn = Type.isUnsigned();
5875 bool quad = Type.isQuad();
5876
5877
5878 switch (BuiltinID) {
5879 default: break;
5880 case NEON::BI__builtin_neon_vabsh_f16:
5883 case NEON::BI__builtin_neon_vaddq_p128: {
5886 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5887 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5888 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
5889 llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
5890 return Builder.CreateBitCast(Ops[0], Int128Ty);
5891 }
5892 case NEON::BI__builtin_neon_vldrq_p128: {
5893 llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
5895 return Builder.CreateAlignedLoad(Int128Ty, Ptr,
5897 }
5898 case NEON::BI__builtin_neon_vstrq_p128: {
5899 Value *Ptr = Ops[0];
5901 }
5902 case NEON::BI__builtin_neon_vcvts_f32_u32:
5903 case NEON::BI__builtin_neon_vcvtd_f64_u64:
5904 usgn = true;
5905 [[fallthrough]];
5906 case NEON::BI__builtin_neon_vcvts_f32_s32:
5907 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
5909 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5912 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
5913 if (usgn)
5914 return Builder.CreateUIToFP(Ops[0], FTy);
5915 return Builder.CreateSIToFP(Ops[0], FTy);
5916 }
5917 case NEON::BI__builtin_neon_vcvth_f16_u16:
5918 case NEON::BI__builtin_neon_vcvth_f16_u32:
5919 case NEON::BI__builtin_neon_vcvth_f16_u64:
5920 usgn = true;
5921 [[fallthrough]];
5922 case NEON::BI__builtin_neon_vcvth_f16_s16:
5923 case NEON::BI__builtin_neon_vcvth_f16_s32:
5924 case NEON::BI__builtin_neon_vcvth_f16_s64: {
5926 llvm::Type *FTy = HalfTy;
5927 llvm::Type *InTy;
5928 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
5930 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
5932 else
5934 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
5935 if (usgn)
5936 return Builder.CreateUIToFP(Ops[0], FTy);
5937 return Builder.CreateSIToFP(Ops[0], FTy);
5938 }
5939 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5940 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5941 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5942 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5943 case NEON::BI__builtin_neon_vcvth_u16_f16:
5944 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5945 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5946 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5947 case NEON::BI__builtin_neon_vcvtph_s16_f16:
5948 case NEON::BI__builtin_neon_vcvth_s16_f16: {
5949 unsigned Int;
5950 llvm::Type *InTy = Int16Ty;
5951 llvm::Type* FTy = HalfTy;
5952 llvm::Type *Tys[2] = {InTy, FTy};
5954 switch (BuiltinID) {
5955 default: llvm_unreachable("missing builtin ID in switch!");
5956 case NEON::BI__builtin_neon_vcvtah_u16_f16:
5957 Int = Intrinsic::aarch64_neon_fcvtau; break;
5958 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
5959 Int = Intrinsic::aarch64_neon_fcvtmu; break;
5960 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
5961 Int = Intrinsic::aarch64_neon_fcvtnu; break;
5962 case NEON::BI__builtin_neon_vcvtph_u16_f16:
5963 Int = Intrinsic::aarch64_neon_fcvtpu; break;
5964 case NEON::BI__builtin_neon_vcvth_u16_f16:
5965 Int = Intrinsic::aarch64_neon_fcvtzu; break;
5966 case NEON::BI__builtin_neon_vcvtah_s16_f16:
5967 Int = Intrinsic::aarch64_neon_fcvtas; break;
5968 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
5969 Int = Intrinsic::aarch64_neon_fcvtms; break;
5970 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
5971 Int = Intrinsic::aarch64_neon_fcvtns; break;
5972 case NEON::BI__builtin_neon_vcvtph_s16_f16:
5973 Int = Intrinsic::aarch64_neon_fcvtps; break;
5974 case NEON::BI__builtin_neon_vcvth_s16_f16:
5975 Int = Intrinsic::aarch64_neon_fcvtzs; break;
5976 }
5977 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt");
5978 }
5979 case NEON::BI__builtin_neon_vcaleh_f16:
5980 case NEON::BI__builtin_neon_vcalth_f16:
5981 case NEON::BI__builtin_neon_vcageh_f16:
5982 case NEON::BI__builtin_neon_vcagth_f16: {
5983 unsigned Int;
5984 llvm::Type* InTy = Int32Ty;
5985 llvm::Type* FTy = HalfTy;
5986 llvm::Type *Tys[2] = {InTy, FTy};
5988 switch (BuiltinID) {
5989 default: llvm_unreachable("missing builtin ID in switch!");
5990 case NEON::BI__builtin_neon_vcageh_f16:
5991 Int = Intrinsic::aarch64_neon_facge; break;
5992 case NEON::BI__builtin_neon_vcagth_f16:
5993 Int = Intrinsic::aarch64_neon_facgt; break;
5994 case NEON::BI__builtin_neon_vcaleh_f16:
5995 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break;
5996 case NEON::BI__builtin_neon_vcalth_f16:
5997 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break;
5998 }
5999 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "facg");
6001 }
6002 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
6003 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
6004 unsigned Int;
6005 llvm::Type* InTy = Int32Ty;
6006 llvm::Type* FTy = HalfTy;
6007 llvm::Type *Tys[2] = {InTy, FTy};
6009 switch (BuiltinID) {
6010 default: llvm_unreachable("missing builtin ID in switch!");
6011 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
6012 Int = Intrinsic::aarch64_neon_vcvtfp2fxs; break;
6013 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
6014 Int = Intrinsic::aarch64_neon_vcvtfp2fxu; break;
6015 }
6016 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
6018 }
6019 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
6020 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
6021 unsigned Int;
6022 llvm::Type* FTy = HalfTy;
6023 llvm::Type* InTy = Int32Ty;
6024 llvm::Type *Tys[2] = {FTy, InTy};
6026 switch (BuiltinID) {
6027 default: llvm_unreachable("missing builtin ID in switch!");
6028 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
6029 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
6030 Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext");
6031 break;
6032 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
6033 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
6034 Ops[0] = Builder.CreateZExt(Ops[0], InTy);
6035 break;
6036 }
6037 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
6038 }
6039 case NEON::BI__builtin_neon_vpaddd_s64: {
6040 auto *Ty = llvm::FixedVectorType::get(Int64Ty, 2);
6042
6043 Vec = Builder.CreateBitCast(Vec, Ty, "v2i64");
6044 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
6045 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
6046 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
6047 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
6048
6049 return Builder.CreateAdd(Op0, Op1, "vpaddd");
6050 }
6051 case NEON::BI__builtin_neon_vpaddd_f64: {
6052 auto *Ty = llvm::FixedVectorType::get(DoubleTy, 2);
6054
6055 Vec = Builder.CreateBitCast(Vec, Ty, "v2f64");
6056 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
6057 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
6058 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
6059 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
6060
6061 return Builder.CreateFAdd(Op0, Op1, "vpaddd");
6062 }
6063 case NEON::BI__builtin_neon_vpadds_f32: {
6064 auto *Ty = llvm::FixedVectorType::get(FloatTy, 2);
6066
6067 Vec = Builder.CreateBitCast(Vec, Ty, "v2f32");
6068 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
6069 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
6070 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
6071 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
6072
6073 return Builder.CreateFAdd(Op0, Op1, "vpaddd");
6074 }
6075 case NEON::BI__builtin_neon_vceqzd_s64:
6079 ICmpInst::ICMP_EQ, "vceqz");
6080 case NEON::BI__builtin_neon_vceqzd_f64:
6081 case NEON::BI__builtin_neon_vceqzs_f32:
6082 case NEON::BI__builtin_neon_vceqzh_f16:
6086 ICmpInst::FCMP_OEQ, "vceqz");
6087 case NEON::BI__builtin_neon_vcgezd_s64:
6091 ICmpInst::ICMP_SGE, "vcgez");
6092 case NEON::BI__builtin_neon_vcgezd_f64:
6093 case NEON::BI__builtin_neon_vcgezs_f32:
6094 case NEON::BI__builtin_neon_vcgezh_f16:
6098 ICmpInst::FCMP_OGE, "vcgez");
6099 case NEON::BI__builtin_neon_vclezd_s64:
6103 ICmpInst::ICMP_SLE, "vclez");
6104 case NEON::BI__builtin_neon_vclezd_f64:
6105 case NEON::BI__builtin_neon_vclezs_f32:
6106 case NEON::BI__builtin_neon_vclezh_f16:
6110 ICmpInst::FCMP_OLE, "vclez");
6111 case NEON::BI__builtin_neon_vcgtzd_s64:
6115 ICmpInst::ICMP_SGT, "vcgtz");
6116 case NEON::BI__builtin_neon_vcgtzd_f64:
6117 case NEON::BI__builtin_neon_vcgtzs_f32:
6118 case NEON::BI__builtin_neon_vcgtzh_f16:
6122 ICmpInst::FCMP_OGT, "vcgtz");
6123 case NEON::BI__builtin_neon_vcltzd_s64:
6127 ICmpInst::ICMP_SLT, "vcltz");
6128
6129 case NEON::BI__builtin_neon_vcltzd_f64:
6130 case NEON::BI__builtin_neon_vcltzs_f32:
6131 case NEON::BI__builtin_neon_vcltzh_f16:
6135 ICmpInst::FCMP_OLT, "vcltz");
6136
6137 case NEON::BI__builtin_neon_vceqzd_u64: {
6140 Ops[0] =
6141 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty));
6142 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd");
6143 }
6144 case NEON::BI__builtin_neon_vceqd_f64:
6145 case NEON::BI__builtin_neon_vcled_f64:
6146 case NEON::BI__builtin_neon_vcltd_f64:
6147 case NEON::BI__builtin_neon_vcged_f64:
6148 case NEON::BI__builtin_neon_vcgtd_f64: {
6149 llvm::CmpInst::Predicate P;
6150 switch (BuiltinID) {
6151 default: llvm_unreachable("missing builtin ID in switch!");
6152 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break;
6153 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break;
6154 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break;
6155 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break;
6156 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break;
6157 }
6161 if (P == llvm::FCmpInst::FCMP_OEQ)
6162 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
6163 else
6164 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
6165 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd");
6166 }
6167 case NEON::BI__builtin_neon_vceqs_f32:
6168 case NEON::BI__builtin_neon_vcles_f32:
6169 case NEON::BI__builtin_neon_vclts_f32:
6170 case NEON::BI__builtin_neon_vcges_f32:
6171 case NEON::BI__builtin_neon_vcgts_f32: {
6172 llvm::CmpInst::Predicate P;
6173 switch (BuiltinID) {
6174 default: llvm_unreachable("missing builtin ID in switch!");
6175 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break;
6176 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break;
6177 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break;
6178 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break;
6179 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break;
6180 }
6184 if (P == llvm::FCmpInst::FCMP_OEQ)
6185 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
6186 else
6187 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
6188 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd");
6189 }
6190 case NEON::BI__builtin_neon_vceqh_f16:
6191 case NEON::BI__builtin_neon_vcleh_f16:
6192 case NEON::BI__builtin_neon_vclth_f16:
6193 case NEON::BI__builtin_neon_vcgeh_f16:
6194 case NEON::BI__builtin_neon_vcgth_f16: {
6195 llvm::CmpInst::Predicate P;
6196 switch (BuiltinID) {
6197 default: llvm_unreachable("missing builtin ID in switch!");
6198 case NEON::BI__builtin_neon_vceqh_f16: P = llvm::FCmpInst::FCMP_OEQ; break;
6199 case NEON::BI__builtin_neon_vcleh_f16: P = llvm::FCmpInst::FCMP_OLE; break;
6200 case NEON::BI__builtin_neon_vclth_f16: P = llvm::FCmpInst::FCMP_OLT; break;
6201 case NEON::BI__builtin_neon_vcgeh_f16: P = llvm::FCmpInst::FCMP_OGE; break;
6202 case NEON::BI__builtin_neon_vcgth_f16: P = llvm::FCmpInst::FCMP_OGT; break;
6203 }
6207 if (P == llvm::FCmpInst::FCMP_OEQ)
6208 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
6209 else
6210 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
6211 return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd");
6212 }
6213 case NEON::BI__builtin_neon_vceqd_s64:
6214 case NEON::BI__builtin_neon_vceqd_u64:
6215 case NEON::BI__builtin_neon_vcgtd_s64:
6216 case NEON::BI__builtin_neon_vcgtd_u64:
6217 case NEON::BI__builtin_neon_vcltd_s64:
6218 case NEON::BI__builtin_neon_vcltd_u64:
6219 case NEON::BI__builtin_neon_vcged_u64:
6220 case NEON::BI__builtin_neon_vcged_s64:
6221 case NEON::BI__builtin_neon_vcled_u64:
6222 case NEON::BI__builtin_neon_vcled_s64: {
6223 llvm::CmpInst::Predicate P;
6224 switch (BuiltinID) {
6225 default: llvm_unreachable("missing builtin ID in switch!");
6226 case NEON::BI__builtin_neon_vceqd_s64:
6227 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break;
6228 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break;
6229 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break;
6230 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break;
6231 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break;
6232 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break;
6233 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break;
6234 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break;
6235 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break;
6236 }
6240 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
6241 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd");
6242 }
6243 case NEON::BI__builtin_neon_vtstd_s64:
6244 case NEON::BI__builtin_neon_vtstd_u64: {
6248 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
6249 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
6250 llvm::Constant::getNullValue(Int64Ty));
6251 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd");
6252 }
6253 case NEON::BI__builtin_neon_vset_lane_i8:
6254 case NEON::BI__builtin_neon_vset_lane_i16:
6255 case NEON::BI__builtin_neon_vset_lane_i32:
6256 case NEON::BI__builtin_neon_vset_lane_i64:
6257 case NEON::BI__builtin_neon_vset_lane_bf16:
6258 case NEON::BI__builtin_neon_vset_lane_f32:
6259 case NEON::BI__builtin_neon_vsetq_lane_i8:
6260 case NEON::BI__builtin_neon_vsetq_lane_i16:
6261 case NEON::BI__builtin_neon_vsetq_lane_i32:
6262 case NEON::BI__builtin_neon_vsetq_lane_i64:
6263 case NEON::BI__builtin_neon_vsetq_lane_bf16:
6264 case NEON::BI__builtin_neon_vsetq_lane_f32:
6266 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
6267 case NEON::BI__builtin_neon_vset_lane_f64:
6268
6269 Ops[1] =
6270 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 1));
6272 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
6273 case NEON::BI__builtin_neon_vset_lane_mf8:
6274 case NEON::BI__builtin_neon_vsetq_lane_mf8:
6276
6277 Ops[0] =
6279 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
6280 case NEON::BI__builtin_neon_vsetq_lane_f64:
6281
6282 Ops[1] =
6283 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 2));
6285 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
6286
6287 case NEON::BI__builtin_neon_vget_lane_i8:
6288 case NEON::BI__builtin_neon_vdupb_lane_i8:
6289 Ops[0] =
6290 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 8));
6292 "vget_lane");
6293 case NEON::BI__builtin_neon_vgetq_lane_i8:
6294 case NEON::BI__builtin_neon_vdupb_laneq_i8:
6295 Ops[0] =
6296 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 16));
6298 "vgetq_lane");
6299 case NEON::BI__builtin_neon_vget_lane_mf8:
6300 case NEON::BI__builtin_neon_vdupb_lane_mf8:
6301 case NEON::BI__builtin_neon_vgetq_lane_mf8:
6302 case NEON::BI__builtin_neon_vdupb_laneq_mf8:
6304 "vget_lane");
6305 case NEON::BI__builtin_neon_vget_lane_i16:
6306 case NEON::BI__builtin_neon_vduph_lane_i16:
6307 Ops[0] =
6308 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 4));
6310 "vget_lane");
6311 case NEON::BI__builtin_neon_vgetq_lane_i16:
6312 case NEON::BI__builtin_neon_vduph_laneq_i16:
6313 Ops[0] =
6314 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 8));
6316 "vgetq_lane");
6317 case NEON::BI__builtin_neon_vget_lane_i32:
6318 case NEON::BI__builtin_neon_vdups_lane_i32:
6319 Ops[0] =
6320 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 2));
6322 "vget_lane");
6323 case NEON::BI__builtin_neon_vdups_lane_f32:
6324 Ops[0] =
6325 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2));
6327 "vdups_lane");
6328 case NEON::BI__builtin_neon_vgetq_lane_i32:
6329 case NEON::BI__builtin_neon_vdups_laneq_i32:
6330 Ops[0] =
6331 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 4));
6333 "vgetq_lane");
6334 case NEON::BI__builtin_neon_vget_lane_i64:
6335 case NEON::BI__builtin_neon_vdupd_lane_i64:
6336 Ops[0] =
6337 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 1));
6339 "vget_lane");
6340 case NEON::BI__builtin_neon_vdupd_lane_f64:
6341 Ops[0] =
6342 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1));
6344 "vdupd_lane");
6345 case NEON::BI__builtin_neon_vgetq_lane_i64:
6346 case NEON::BI__builtin_neon_vdupd_laneq_i64:
6347 Ops[0] =
6348 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2));
6350 "vgetq_lane");
6351 case NEON::BI__builtin_neon_vget_lane_f32:
6352 Ops[0] =
6353 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2));
6355 "vget_lane");
6356 case NEON::BI__builtin_neon_vget_lane_f64:
6357 Ops[0] =
6358 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1));
6360 "vget_lane");
6361 case NEON::BI__builtin_neon_vgetq_lane_f32:
6362 case NEON::BI__builtin_neon_vdups_laneq_f32:
6363 Ops[0] =
6364 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 4));
6366 "vgetq_lane");
6367 case NEON::BI__builtin_neon_vgetq_lane_f64:
6368 case NEON::BI__builtin_neon_vdupd_laneq_f64:
6369 Ops[0] =
6370 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 2));
6372 "vgetq_lane");
6373 case NEON::BI__builtin_neon_vaddh_f16:
6375 return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh");
6376 case NEON::BI__builtin_neon_vsubh_f16:
6378 return Builder.CreateFSub(Ops[0], Ops[1], "vsubh");
6379 case NEON::BI__builtin_neon_vmulh_f16:
6381 return Builder.CreateFMul(Ops[0], Ops[1], "vmulh");
6382 case NEON::BI__builtin_neon_vdivh_f16:
6384 return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh");
6385 case NEON::BI__builtin_neon_vfmah_f16:
6386
6388 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, HalfTy,
6390 case NEON::BI__builtin_neon_vfmsh_f16: {
6392
6393
6395 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, HalfTy,
6397 }
6398 case NEON::BI__builtin_neon_vaddd_s64:
6399 case NEON::BI__builtin_neon_vaddd_u64:
6401 case NEON::BI__builtin_neon_vsubd_s64:
6402 case NEON::BI__builtin_neon_vsubd_u64:
6404 case NEON::BI__builtin_neon_vqdmlalh_s16:
6405 case NEON::BI__builtin_neon_vqdmlslh_s16: {
6409 auto *VTy = llvm::FixedVectorType::get(Int32Ty, 4);
6410 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
6411 ProductOps, "vqdmlXl");
6412 Constant *CI = ConstantInt::get(SizeTy, 0);
6413 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
6414
6415 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
6416 ? Intrinsic::aarch64_neon_sqadd
6417 : Intrinsic::aarch64_neon_sqsub;
6419 }
6420 case NEON::BI__builtin_neon_vqshlud_n_s64: {
6424 Ops, "vqshlu_n");
6425 }
6426 case NEON::BI__builtin_neon_vqshld_n_u64:
6427 case NEON::BI__builtin_neon_vqshld_n_s64: {
6428 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
6429 ? Intrinsic::aarch64_neon_uqshl
6430 : Intrinsic::aarch64_neon_sqshl;
6434 }
6435 case NEON::BI__builtin_neon_vrshrd_n_u64:
6436 case NEON::BI__builtin_neon_vrshrd_n_s64: {
6437 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
6438 ? Intrinsic::aarch64_neon_urshl
6439 : Intrinsic::aarch64_neon_srshl;
6442 Ops[1] = ConstantInt::get(Int64Ty, -SV);
6444 }
6445 case NEON::BI__builtin_neon_vrsrad_n_u64:
6446 case NEON::BI__builtin_neon_vrsrad_n_s64: {
6447 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
6448 ? Intrinsic::aarch64_neon_urshl
6449 : Intrinsic::aarch64_neon_srshl;
6453 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
6455 }
6456 case NEON::BI__builtin_neon_vshld_n_s64:
6457 case NEON::BI__builtin_neon_vshld_n_u64: {
6459 return Builder.CreateShl(
6460 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n");
6461 }
6462 case NEON::BI__builtin_neon_vshrd_n_s64: {
6464 return Builder.CreateAShr(
6465 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
6466 Amt->getZExtValue())),
6467 "shrd_n");
6468 }
6469 case NEON::BI__builtin_neon_vshrd_n_u64: {
6471 uint64_t ShiftAmt = Amt->getZExtValue();
6472
6473 if (ShiftAmt == 64)
6474 return ConstantInt::get(Int64Ty, 0);
6475 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
6476 "shrd_n");
6477 }
6478 case NEON::BI__builtin_neon_vsrad_n_s64: {
6480 Ops[1] = Builder.CreateAShr(
6481 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
6482 Amt->getZExtValue())),
6483 "shrd_n");
6484 return Builder.CreateAdd(Ops[0], Ops[1]);
6485 }
6486 case NEON::BI__builtin_neon_vsrad_n_u64: {
6488 uint64_t ShiftAmt = Amt->getZExtValue();
6489
6490
6491 if (ShiftAmt == 64)
6492 return Ops[0];
6493 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
6494 "shrd_n");
6495 return Builder.CreateAdd(Ops[0], Ops[1]);
6496 }
6497 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
6498 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
6499 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
6500 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
6502 "lane");
6506 auto *VTy = llvm::FixedVectorType::get(Int32Ty, 4);
6507 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
6508 ProductOps, "vqdmlXl");
6509 Constant *CI = ConstantInt::get(SizeTy, 0);
6510 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
6511 Ops.pop_back();
6512
6513 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
6514 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
6515 ? Intrinsic::aarch64_neon_sqadd
6516 : Intrinsic::aarch64_neon_sqsub;
6518 }
6519 case NEON::BI__builtin_neon_vqdmlals_s32:
6520 case NEON::BI__builtin_neon_vqdmlsls_s32: {
6522 ProductOps.push_back(Ops[1]);
6524 Ops[1] =
6525 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
6526 ProductOps, "vqdmlXl");
6527
6528 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
6529 ? Intrinsic::aarch64_neon_sqadd
6530 : Intrinsic::aarch64_neon_sqsub;
6532 }
6533 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
6534 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
6535 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
6536 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
6538 "lane");
6540 ProductOps.push_back(Ops[1]);
6541 ProductOps.push_back(Ops[2]);
6542 Ops[1] =
6543 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
6544 ProductOps, "vqdmlXl");
6545 Ops.pop_back();
6546
6547 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
6548 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
6549 ? Intrinsic::aarch64_neon_sqadd
6550 : Intrinsic::aarch64_neon_sqsub;
6552 }
6553 case NEON::BI__builtin_neon_vget_lane_bf16:
6554 case NEON::BI__builtin_neon_vduph_lane_bf16:
6555 case NEON::BI__builtin_neon_vduph_lane_f16: {
6557 "vget_lane");
6558 }
6559 case NEON::BI__builtin_neon_vgetq_lane_bf16:
6560 case NEON::BI__builtin_neon_vduph_laneq_bf16:
6561 case NEON::BI__builtin_neon_vduph_laneq_f16: {
6563 "vgetq_lane");
6564 }
6565 case NEON::BI__builtin_neon_vcvt_bf16_f32: {
6566 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6567 llvm::Type *V4BF16 = FixedVectorType::get(Builder.getBFloatTy(), 4);
6568 return Builder.CreateFPTrunc(Builder.CreateBitCast(Ops[0], V4F32), V4BF16);
6569 }
6570 case NEON::BI__builtin_neon_vcvtq_low_bf16_f32: {
6572 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
6573 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6574 llvm::Type *V4BF16 = FixedVectorType::get(Builder.getBFloatTy(), 4);
6575 llvm::Value *Trunc =
6576 Builder.CreateFPTrunc(Builder.CreateBitCast(Ops[0], V4F32), V4BF16);
6577 return Builder.CreateShuffleVector(
6578 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
6579 }
6580 case NEON::BI__builtin_neon_vcvtq_high_bf16_f32: {
6582 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
6584 std::iota(LoMask.begin(), LoMask.end(), 0);
6585 llvm::Type *V4F32 = FixedVectorType::get(Builder.getFloatTy(), 4);
6586 llvm::Type *V4BF16 = FixedVectorType::get(Builder.getBFloatTy(), 4);
6587 llvm::Type *V8BF16 = FixedVectorType::get(Builder.getBFloatTy(), 8);
6588 llvm::Value *Inactive = Builder.CreateShuffleVector(
6589 Builder.CreateBitCast(Ops[0], V8BF16), LoMask);
6590 llvm::Value *Trunc =
6591 Builder.CreateFPTrunc(Builder.CreateBitCast(Ops[1], V4F32), V4BF16);
6592 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
6593 }
6594
6595 case clang::AArch64::BI_InterlockedAdd:
6596 case clang::AArch64::BI_InterlockedAdd_acq:
6597 case clang::AArch64::BI_InterlockedAdd_rel:
6598 case clang::AArch64::BI_InterlockedAdd_nf:
6599 case clang::AArch64::BI_InterlockedAdd64:
6600 case clang::AArch64::BI_InterlockedAdd64_acq:
6601 case clang::AArch64::BI_InterlockedAdd64_rel:
6602 case clang::AArch64::BI_InterlockedAdd64_nf: {
6605 llvm::AtomicOrdering Ordering;
6606 switch (BuiltinID) {
6607 case clang::AArch64::BI_InterlockedAdd:
6608 case clang::AArch64::BI_InterlockedAdd64:
6609 Ordering = llvm::AtomicOrdering::SequentiallyConsistent;
6610 break;
6611 case clang::AArch64::BI_InterlockedAdd_acq:
6612 case clang::AArch64::BI_InterlockedAdd64_acq:
6613 Ordering = llvm::AtomicOrdering::Acquire;
6614 break;
6615 case clang::AArch64::BI_InterlockedAdd_rel:
6616 case clang::AArch64::BI_InterlockedAdd64_rel:
6617 Ordering = llvm::AtomicOrdering::Release;
6618 break;
6619 case clang::AArch64::BI_InterlockedAdd_nf:
6620 case clang::AArch64::BI_InterlockedAdd64_nf:
6621 Ordering = llvm::AtomicOrdering::Monotonic;
6622 break;
6623 default:
6624 llvm_unreachable("missing builtin ID in switch!");
6625 }
6626 AtomicRMWInst *RMWI =
6627 Builder.CreateAtomicRMW(AtomicRMWInst::Add, DestAddr, Val, Ordering);
6628 return Builder.CreateAdd(RMWI, Val);
6629 }
6630 }
6631
6633 llvm::Type *Ty = VTy;
6634 if (!Ty)
6635 return nullptr;
6636
6637
6638
6641
6647
6649 return V;
6650
6651 unsigned Int;
6652 bool ExtractLow = false;
6653 bool ExtendLaneArg = false;
6654 switch (BuiltinID) {
6655 default: return nullptr;
6656 case NEON::BI__builtin_neon_vbsl_v:
6657 case NEON::BI__builtin_neon_vbslq_v: {
6658 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
6659 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl");
6660 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl");
6661 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl");
6662
6663 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
6664 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
6665 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
6666 return Builder.CreateBitCast(Ops[0], Ty);
6667 }
6668 case NEON::BI__builtin_neon_vfma_lane_v:
6669 case NEON::BI__builtin_neon_vfmaq_lane_v: {
6670
6671
6672 Value *Addend = Ops[0];
6673 Value *Multiplicand = Ops[1];
6674 Value *LaneSource = Ops[2];
6675 Ops[0] = Multiplicand;
6676 Ops[1] = LaneSource;
6677 Ops[2] = Addend;
6678
6679
6680 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
6681 ? llvm::FixedVectorType::get(VTy->getElementType(),
6682 VTy->getNumElements() / 2)
6683 : VTy;
6685 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
6686 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
6687 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
6688
6689 Ops.pop_back();
6690 Int = Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
6691 : Intrinsic::fma;
6692 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla");
6693 }
6694 case NEON::BI__builtin_neon_vfma_laneq_v: {
6696
6697 if (VTy && VTy->getElementType() == DoubleTy) {
6700 llvm::FixedVectorType *VTy =
6702 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
6703 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
6706 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
6707 DoubleTy, {Ops[1], Ops[2], Ops[0]});
6709 }
6710 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6711 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6712
6713 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
6714 VTy->getNumElements() * 2);
6715 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
6716 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
6718 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
6719
6721 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6722 {Ops[2], Ops[1], Ops[0]});
6723 }
6724 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
6725 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6726 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6727
6728 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6731 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6732 {Ops[2], Ops[1], Ops[0]});
6733 }
6734 case NEON::BI__builtin_neon_vfmah_lane_f16:
6735 case NEON::BI__builtin_neon_vfmas_lane_f32:
6736 case NEON::BI__builtin_neon_vfmah_laneq_f16:
6737 case NEON::BI__builtin_neon_vfmas_laneq_f32:
6738 case NEON::BI__builtin_neon_vfmad_lane_f64:
6739 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
6742 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
6744 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6745 {Ops[1], Ops[2], Ops[0]});
6746 }
6747 case NEON::BI__builtin_neon_vmull_v:
6748
6749 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
6750 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
6751 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
6752 case NEON::BI__builtin_neon_vmax_v:
6753 case NEON::BI__builtin_neon_vmaxq_v:
6754
6755 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
6756 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
6757 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
6758 case NEON::BI__builtin_neon_vmaxh_f16: {
6760 Int = Intrinsic::aarch64_neon_fmax;
6762 }
6763 case NEON::BI__builtin_neon_vmin_v:
6764 case NEON::BI__builtin_neon_vminq_v:
6765
6766 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
6767 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
6768 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
6769 case NEON::BI__builtin_neon_vminh_f16: {
6771 Int = Intrinsic::aarch64_neon_fmin;
6773 }
6774 case NEON::BI__builtin_neon_vabd_v:
6775 case NEON::BI__builtin_neon_vabdq_v:
6776
6777 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
6778 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
6779 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd");
6780 case NEON::BI__builtin_neon_vpadal_v:
6781 case NEON::BI__builtin_neon_vpadalq_v: {
6782 unsigned ArgElts = VTy->getNumElements();
6783 llvm::IntegerType *EltTy = cast(VTy->getElementType());
6784 unsigned BitWidth = EltTy->getBitWidth();
6785 auto *ArgTy = llvm::FixedVectorType::get(
6786 llvm::IntegerType::get(getLLVMContext(), BitWidth / 2), 2 * ArgElts);
6787 llvm::Type* Tys[2] = { VTy, ArgTy };
6788 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
6790 TmpOps.push_back(Ops[1]);
6791 Function *F = CGM.getIntrinsic(Int, Tys);
6792 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal");
6793 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
6794 return Builder.CreateAdd(tmp, addend);
6795 }
6796 case NEON::BI__builtin_neon_vpmin_v:
6797 case NEON::BI__builtin_neon_vpminq_v:
6798
6799 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
6800 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
6801 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
6802 case NEON::BI__builtin_neon_vpmax_v:
6803 case NEON::BI__builtin_neon_vpmaxq_v:
6804
6805 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
6806 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
6807 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
6808 case NEON::BI__builtin_neon_vminnm_v:
6809 case NEON::BI__builtin_neon_vminnmq_v:
6810 Int = Intrinsic::aarch64_neon_fminnm;
6811 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm");
6812 case NEON::BI__builtin_neon_vminnmh_f16:
6814 Int = Intrinsic::aarch64_neon_fminnm;
6816 case NEON::BI__builtin_neon_vmaxnm_v:
6817 case NEON::BI__builtin_neon_vmaxnmq_v:
6818 Int = Intrinsic::aarch64_neon_fmaxnm;
6819 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm");
6820 case NEON::BI__builtin_neon_vmaxnmh_f16:
6822 Int = Intrinsic::aarch64_neon_fmaxnm;
6824 case NEON::BI__builtin_neon_vrecpss_f32: {
6827 Ops, "vrecps");
6828 }
6829 case NEON::BI__builtin_neon_vrecpsd_f64:
6832 Ops, "vrecps");
6833 case NEON::BI__builtin_neon_vrecpsh_f16:
6836 Ops, "vrecps");
6837 case NEON::BI__builtin_neon_vqshrun_n_v:
6838 Int = Intrinsic::aarch64_neon_sqshrun;
6839 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n");
6840 case NEON::BI__builtin_neon_vqrshrun_n_v:
6841 Int = Intrinsic::aarch64_neon_sqrshrun;
6842 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n");
6843 case NEON::BI__builtin_neon_vqshrn_n_v:
6844 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
6845 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n");
6846 case NEON::BI__builtin_neon_vrshrn_n_v:
6847 Int = Intrinsic::aarch64_neon_rshrn;
6848 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n");
6849 case NEON::BI__builtin_neon_vqrshrn_n_v:
6850 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
6851 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n");
6852 case NEON::BI__builtin_neon_vrndah_f16: {
6854 Int = Builder.getIsFPConstrained()
6855 ? Intrinsic::experimental_constrained_round
6856 : Intrinsic::round;
6858 }
6859 case NEON::BI__builtin_neon_vrnda_v:
6860 case NEON::BI__builtin_neon_vrndaq_v: {
6861 Int = Builder.getIsFPConstrained()
6862 ? Intrinsic::experimental_constrained_round
6863 : Intrinsic::round;
6864 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda");
6865 }
6866 case NEON::BI__builtin_neon_vrndih_f16: {
6868 Int = Builder.getIsFPConstrained()
6869 ? Intrinsic::experimental_constrained_nearbyint
6870 : Intrinsic::nearbyint;
6872 }
6873 case NEON::BI__builtin_neon_vrndmh_f16: {
6875 Int = Builder.getIsFPConstrained()
6876 ? Intrinsic::experimental_constrained_floor
6877 : Intrinsic::floor;
6879 }
6880 case NEON::BI__builtin_neon_vrndm_v:
6881 case NEON::BI__builtin_neon_vrndmq_v: {
6882 Int = Builder.getIsFPConstrained()
6883 ? Intrinsic::experimental_constrained_floor
6884 : Intrinsic::floor;
6885 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm");
6886 }
6887 case NEON::BI__builtin_neon_vrndnh_f16: {
6889 Int = Builder.getIsFPConstrained()
6890 ? Intrinsic::experimental_constrained_roundeven
6891 : Intrinsic::roundeven;
6893 }
6894 case NEON::BI__builtin_neon_vrndn_v:
6895 case NEON::BI__builtin_neon_vrndnq_v: {
6896 Int = Builder.getIsFPConstrained()
6897 ? Intrinsic::experimental_constrained_roundeven
6898 : Intrinsic::roundeven;
6899 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn");
6900 }
6901 case NEON::BI__builtin_neon_vrndns_f32: {
6903 Int = Builder.getIsFPConstrained()
6904 ? Intrinsic::experimental_constrained_roundeven
6905 : Intrinsic::roundeven;
6907 }
6908 case NEON::BI__builtin_neon_vrndph_f16: {
6910 Int = Builder.getIsFPConstrained()
6911 ? Intrinsic::experimental_constrained_ceil
6912 : Intrinsic::ceil;
6914 }
6915 case NEON::BI__builtin_neon_vrndp_v:
6916 case NEON::BI__builtin_neon_vrndpq_v: {
6917 Int = Builder.getIsFPConstrained()
6918 ? Intrinsic::experimental_constrained_ceil
6919 : Intrinsic::ceil;
6920 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp");
6921 }
6922 case NEON::BI__builtin_neon_vrndxh_f16: {
6924 Int = Builder.getIsFPConstrained()
6925 ? Intrinsic::experimental_constrained_rint
6926 : Intrinsic::rint;
6928 }
6929 case NEON::BI__builtin_neon_vrndx_v:
6930 case NEON::BI__builtin_neon_vrndxq_v: {
6931 Int = Builder.getIsFPConstrained()
6932 ? Intrinsic::experimental_constrained_rint
6933 : Intrinsic::rint;
6934 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx");
6935 }
6936 case NEON::BI__builtin_neon_vrndh_f16: {
6938 Int = Builder.getIsFPConstrained()
6939 ? Intrinsic::experimental_constrained_trunc
6940 : Intrinsic::trunc;
6942 }
6943 case NEON::BI__builtin_neon_vrnd32x_f32:
6944 case NEON::BI__builtin_neon_vrnd32xq_f32:
6945 case NEON::BI__builtin_neon_vrnd32x_f64:
6946 case NEON::BI__builtin_neon_vrnd32xq_f64: {
6948 Int = Intrinsic::aarch64_neon_frint32x;
6949 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd32x");
6950 }
6951 case NEON::BI__builtin_neon_vrnd32z_f32:
6952 case NEON::BI__builtin_neon_vrnd32zq_f32:
6953 case NEON::BI__builtin_neon_vrnd32z_f64:
6954 case NEON::BI__builtin_neon_vrnd32zq_f64: {
6956 Int = Intrinsic::aarch64_neon_frint32z;
6957 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd32z");
6958 }
6959 case NEON::BI__builtin_neon_vrnd64x_f32:
6960 case NEON::BI__builtin_neon_vrnd64xq_f32:
6961 case NEON::BI__builtin_neon_vrnd64x_f64:
6962 case NEON::BI__builtin_neon_vrnd64xq_f64: {
6964 Int = Intrinsic::aarch64_neon_frint64x;
6965 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd64x");
6966 }
6967 case NEON::BI__builtin_neon_vrnd64z_f32:
6968 case NEON::BI__builtin_neon_vrnd64zq_f32:
6969 case NEON::BI__builtin_neon_vrnd64z_f64:
6970 case NEON::BI__builtin_neon_vrnd64zq_f64: {
6972 Int = Intrinsic::aarch64_neon_frint64z;
6973 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd64z");
6974 }
6975 case NEON::BI__builtin_neon_vrnd_v:
6976 case NEON::BI__builtin_neon_vrndq_v: {
6977 Int = Builder.getIsFPConstrained()
6978 ? Intrinsic::experimental_constrained_trunc
6979 : Intrinsic::trunc;
6980 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz");
6981 }
6982 case NEON::BI__builtin_neon_vcvt_f64_v:
6983 case NEON::BI__builtin_neon_vcvtq_f64_v:
6984 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6986 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
6987 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
6988 case NEON::BI__builtin_neon_vcvt_f64_f32: {
6990 "unexpected vcvt_f64_f32 builtin");
6993
6994 return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
6995 }
6996 case NEON::BI__builtin_neon_vcvt_f32_f64: {
6998 "unexpected vcvt_f32_f64 builtin");
7001
7002 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
7003 }
7004 case NEON::BI__builtin_neon_vcvt_s32_v:
7005 case NEON::BI__builtin_neon_vcvt_u32_v:
7006 case NEON::BI__builtin_neon_vcvt_s64_v:
7007 case NEON::BI__builtin_neon_vcvt_u64_v:
7008 case NEON::BI__builtin_neon_vcvt_s16_f16:
7009 case NEON::BI__builtin_neon_vcvt_u16_f16:
7010 case NEON::BI__builtin_neon_vcvtq_s32_v:
7011 case NEON::BI__builtin_neon_vcvtq_u32_v:
7012 case NEON::BI__builtin_neon_vcvtq_s64_v:
7013 case NEON::BI__builtin_neon_vcvtq_u64_v:
7014 case NEON::BI__builtin_neon_vcvtq_s16_f16:
7015 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
7016 Int =
7017 usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
7019 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtz");
7020 }
7021 case NEON::BI__builtin_neon_vcvta_s16_f16:
7022 case NEON::BI__builtin_neon_vcvta_u16_f16:
7023 case NEON::BI__builtin_neon_vcvta_s32_v:
7024 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
7025 case NEON::BI__builtin_neon_vcvtaq_s32_v:
7026 case NEON::BI__builtin_neon_vcvta_u32_v:
7027 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
7028 case NEON::BI__builtin_neon_vcvtaq_u32_v:
7029 case NEON::BI__builtin_neon_vcvta_s64_v:
7030 case NEON::BI__builtin_neon_vcvtaq_s64_v:
7031 case NEON::BI__builtin_neon_vcvta_u64_v:
7032 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
7033 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
7035 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta");
7036 }
7037 case NEON::BI__builtin_neon_vcvtm_s16_f16:
7038 case NEON::BI__builtin_neon_vcvtm_s32_v:
7039 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
7040 case NEON::BI__builtin_neon_vcvtmq_s32_v:
7041 case NEON::BI__builtin_neon_vcvtm_u16_f16:
7042 case NEON::BI__builtin_neon_vcvtm_u32_v:
7043 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
7044 case NEON::BI__builtin_neon_vcvtmq_u32_v:
7045 case NEON::BI__builtin_neon_vcvtm_s64_v:
7046 case NEON::BI__builtin_neon_vcvtmq_s64_v:
7047 case NEON::BI__builtin_neon_vcvtm_u64_v:
7048 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
7049 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
7051 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm");
7052 }
7053 case NEON::BI__builtin_neon_vcvtn_s16_f16:
7054 case NEON::BI__builtin_neon_vcvtn_s32_v:
7055 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
7056 case NEON::BI__builtin_neon_vcvtnq_s32_v:
7057 case NEON::BI__builtin_neon_vcvtn_u16_f16:
7058 case NEON::BI__builtin_neon_vcvtn_u32_v:
7059 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
7060 case NEON::BI__builtin_neon_vcvtnq_u32_v:
7061 case NEON::BI__builtin_neon_vcvtn_s64_v:
7062 case NEON::BI__builtin_neon_vcvtnq_s64_v:
7063 case NEON::BI__builtin_neon_vcvtn_u64_v:
7064 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
7065 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
7067 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn");
7068 }
7069 case NEON::BI__builtin_neon_vcvtp_s16_f16:
7070 case NEON::BI__builtin_neon_vcvtp_s32_v:
7071 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
7072 case NEON::BI__builtin_neon_vcvtpq_s32_v:
7073 case NEON::BI__builtin_neon_vcvtp_u16_f16:
7074 case NEON::BI__builtin_neon_vcvtp_u32_v:
7075 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
7076 case NEON::BI__builtin_neon_vcvtpq_u32_v:
7077 case NEON::BI__builtin_neon_vcvtp_s64_v:
7078 case NEON::BI__builtin_neon_vcvtpq_s64_v:
7079 case NEON::BI__builtin_neon_vcvtp_u64_v:
7080 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
7081 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
7083 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp");
7084 }
7085 case NEON::BI__builtin_neon_vmulx_v:
7086 case NEON::BI__builtin_neon_vmulxq_v: {
7087 Int = Intrinsic::aarch64_neon_fmulx;
7088 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx");
7089 }
7090 case NEON::BI__builtin_neon_vmulxh_lane_f16:
7091 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
7092
7093
7095 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
7096 Ops.pop_back();
7097 Int = Intrinsic::aarch64_neon_fmulx;
7099 }
7100 case NEON::BI__builtin_neon_vmul_lane_v:
7101 case NEON::BI__builtin_neon_vmul_laneq_v: {
7102
7103 bool Quad = false;
7104 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
7105 Quad = true;
7107 llvm::FixedVectorType *VTy =
7109 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
7110 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
7113 }
7114 case NEON::BI__builtin_neon_vnegd_s64:
7116 case NEON::BI__builtin_neon_vnegh_f16:
7118 case NEON::BI__builtin_neon_vpmaxnm_v:
7119 case NEON::BI__builtin_neon_vpmaxnmq_v: {
7120 Int = Intrinsic::aarch64_neon_fmaxnmp;
7121 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm");
7122 }
7123 case NEON::BI__builtin_neon_vpminnm_v:
7124 case NEON::BI__builtin_neon_vpminnmq_v: {
7125 Int = Intrinsic::aarch64_neon_fminnmp;
7126 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm");
7127 }
7128 case NEON::BI__builtin_neon_vsqrth_f16: {
7130 Int = Builder.getIsFPConstrained()
7131 ? Intrinsic::experimental_constrained_sqrt
7132 : Intrinsic::sqrt;
7134 }
7135 case NEON::BI__builtin_neon_vsqrt_v:
7136 case NEON::BI__builtin_neon_vsqrtq_v: {
7137 Int = Builder.getIsFPConstrained()
7138 ? Intrinsic::experimental_constrained_sqrt
7139 : Intrinsic::sqrt;
7140 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7141 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt");
7142 }
7143 case NEON::BI__builtin_neon_vrbit_v:
7144 case NEON::BI__builtin_neon_vrbitq_v: {
7145 Int = Intrinsic::bitreverse;
7146 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit");
7147 }
7148 case NEON::BI__builtin_neon_vmaxv_f16: {
7149 Int = Intrinsic::aarch64_neon_fmaxv;
7151 VTy = llvm::FixedVectorType::get(HalfTy, 4);
7152 llvm::Type *Tys[2] = { Ty, VTy };
7154 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
7156 }
7157 case NEON::BI__builtin_neon_vmaxvq_f16: {
7158 Int = Intrinsic::aarch64_neon_fmaxv;
7160 VTy = llvm::FixedVectorType::get(HalfTy, 8);
7161 llvm::Type *Tys[2] = { Ty, VTy };
7163 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
7165 }
7166 case NEON::BI__builtin_neon_vminv_f16: {
7167 Int = Intrinsic::aarch64_neon_fminv;
7169 VTy = llvm::FixedVectorType::get(HalfTy, 4);
7170 llvm::Type *Tys[2] = { Ty, VTy };
7172 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
7174 }
7175 case NEON::BI__builtin_neon_vminvq_f16: {
7176 Int = Intrinsic::aarch64_neon_fminv;
7178 VTy = llvm::FixedVectorType::get(HalfTy, 8);
7179 llvm::Type *Tys[2] = { Ty, VTy };
7181 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
7183 }
7184 case NEON::BI__builtin_neon_vmaxnmv_f16: {
7185 Int = Intrinsic::aarch64_neon_fmaxnmv;
7187 VTy = llvm::FixedVectorType::get(HalfTy, 4);
7188 llvm::Type *Tys[2] = { Ty, VTy };
7190 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
7192 }
7193 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
7194 Int = Intrinsic::aarch64_neon_fmaxnmv;
7196 VTy = llvm::FixedVectorType::get(HalfTy, 8);
7197 llvm::Type *Tys[2] = { Ty, VTy };
7199 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
7201 }
7202 case NEON::BI__builtin_neon_vminnmv_f16: {
7203 Int = Intrinsic::aarch64_neon_fminnmv;
7205 VTy = llvm::FixedVectorType::get(HalfTy, 4);
7206 llvm::Type *Tys[2] = { Ty, VTy };
7208 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
7210 }
7211 case NEON::BI__builtin_neon_vminnmvq_f16: {
7212 Int = Intrinsic::aarch64_neon_fminnmv;
7214 VTy = llvm::FixedVectorType::get(HalfTy, 8);
7215 llvm::Type *Tys[2] = { Ty, VTy };
7217 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
7219 }
7220 case NEON::BI__builtin_neon_vmul_n_f64: {
7223 return Builder.CreateFMul(Ops[0], RHS);
7224 }
7225 case NEON::BI__builtin_neon_vaddlv_u8: {
7226 Int = Intrinsic::aarch64_neon_uaddlv;
7228 VTy = llvm::FixedVectorType::get(Int8Ty, 8);
7229 llvm::Type *Tys[2] = { Ty, VTy };
7231 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
7233 }
7234 case NEON::BI__builtin_neon_vaddlv_u16: {
7235 Int = Intrinsic::aarch64_neon_uaddlv;
7237 VTy = llvm::FixedVectorType::get(Int16Ty, 4);
7238 llvm::Type *Tys[2] = { Ty, VTy };
7240 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
7241 }
7242 case NEON::BI__builtin_neon_vaddlvq_u8: {
7243 Int = Intrinsic::aarch64_neon_uaddlv;
7245 VTy = llvm::FixedVectorType::get(Int8Ty, 16);
7246 llvm::Type *Tys[2] = { Ty, VTy };
7248 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
7250 }
7251 case NEON::BI__builtin_neon_vaddlvq_u16: {
7252 Int = Intrinsic::aarch64_neon_uaddlv;
7254 VTy = llvm::FixedVectorType::get(Int16Ty, 8);
7255 llvm::Type *Tys[2] = { Ty, VTy };
7257 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
7258 }
7259 case NEON::BI__builtin_neon_vaddlv_s8: {
7260 Int = Intrinsic::aarch64_neon_saddlv;
7262 VTy = llvm::FixedVectorType::get(Int8Ty, 8);
7263 llvm::Type *Tys[2] = { Ty, VTy };
7265 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
7267 }
7268 case NEON::BI__builtin_neon_vaddlv_s16: {
7269 Int = Intrinsic::aarch64_neon_saddlv;
7271 VTy = llvm::FixedVectorType::get(Int16Ty, 4);
7272 llvm::Type *Tys[2] = { Ty, VTy };
7274 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
7275 }
7276 case NEON::BI__builtin_neon_vaddlvq_s8: {
7277 Int = Intrinsic::aarch64_neon_saddlv;
7279 VTy = llvm::FixedVectorType::get(Int8Ty, 16);
7280 llvm::Type *Tys[2] = { Ty, VTy };
7282 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
7284 }
7285 case NEON::BI__builtin_neon_vaddlvq_s16: {
7286 Int = Intrinsic::aarch64_neon_saddlv;
7288 VTy = llvm::FixedVectorType::get(Int16Ty, 8);
7289 llvm::Type *Tys[2] = { Ty, VTy };
7291 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
7292 }
7293 case NEON::BI__builtin_neon_vsri_n_v:
7294 case NEON::BI__builtin_neon_vsriq_n_v: {
7295 Int = Intrinsic::aarch64_neon_vsri;
7296 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
7298 }
7299 case NEON::BI__builtin_neon_vsli_n_v:
7300 case NEON::BI__builtin_neon_vsliq_n_v: {
7301 Int = Intrinsic::aarch64_neon_vsli;
7302 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
7304 }
7305 case NEON::BI__builtin_neon_vsra_n_v:
7306 case NEON::BI__builtin_neon_vsraq_n_v:
7307 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7309 return Builder.CreateAdd(Ops[0], Ops[1]);
7310 case NEON::BI__builtin_neon_vrsra_n_v:
7311 case NEON::BI__builtin_neon_vrsraq_n_v: {
7312 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
7314 TmpOps.push_back(Ops[1]);
7315 TmpOps.push_back(Ops[2]);
7316 Function* F = CGM.getIntrinsic(Int, Ty);
7317 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true);
7318 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
7319 return Builder.CreateAdd(Ops[0], tmp);
7320 }
7321 case NEON::BI__builtin_neon_vld1_v:
7322 case NEON::BI__builtin_neon_vld1q_v: {
7324 }
7325 case NEON::BI__builtin_neon_vst1_v:
7326 case NEON::BI__builtin_neon_vst1q_v:
7327 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
7328 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment());
7329 case NEON::BI__builtin_neon_vld1_lane_v:
7330 case NEON::BI__builtin_neon_vld1q_lane_v: {
7331 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7332 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
7334 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
7335 }
7336 case NEON::BI__builtin_neon_vldap1_lane_s64:
7337 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
7338 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7339 llvm::LoadInst *LI = Builder.CreateAlignedLoad(
7340 VTy->getElementType(), Ops[0], PtrOp0.getAlignment());
7341 LI->setAtomic(llvm::AtomicOrdering::Acquire);
7342 Ops[0] = LI;
7343 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vldap1_lane");
7344 }
7345 case NEON::BI__builtin_neon_vld1_dup_v:
7346 case NEON::BI__builtin_neon_vld1q_dup_v: {
7347 Value *V = PoisonValue::get(Ty);
7348 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
7350 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
7351 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
7353 }
7354 case NEON::BI__builtin_neon_vst1_lane_v:
7355 case NEON::BI__builtin_neon_vst1q_lane_v:
7356 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7357 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
7358 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment());
7359 case NEON::BI__builtin_neon_vstl1_lane_s64:
7360 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
7361 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7362 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
7363 llvm::StoreInst *SI =
7365 SI->setAtomic(llvm::AtomicOrdering::Release);
7366 return SI;
7367 }
7368 case NEON::BI__builtin_neon_vld2_v:
7369 case NEON::BI__builtin_neon_vld2q_v: {
7371 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
7372 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
7373 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7374 }
7375 case NEON::BI__builtin_neon_vld3_v:
7376 case NEON::BI__builtin_neon_vld3q_v: {
7378 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
7379 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
7380 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7381 }
7382 case NEON::BI__builtin_neon_vld4_v:
7383 case NEON::BI__builtin_neon_vld4q_v: {
7385 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
7386 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
7387 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7388 }
7389 case NEON::BI__builtin_neon_vld2_dup_v:
7390 case NEON::BI__builtin_neon_vld2q_dup_v: {
7392 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
7393 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
7394 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7395 }
7396 case NEON::BI__builtin_neon_vld3_dup_v:
7397 case NEON::BI__builtin_neon_vld3q_dup_v: {
7399 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
7400 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
7401 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7402 }
7403 case NEON::BI__builtin_neon_vld4_dup_v:
7404 case NEON::BI__builtin_neon_vld4q_dup_v: {
7406 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
7407 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
7408 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7409 }
7410 case NEON::BI__builtin_neon_vld2_lane_v:
7411 case NEON::BI__builtin_neon_vld2q_lane_v: {
7412 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
7413 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
7414 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
7415 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7416 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7418 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld2_lane");
7419 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7420 }
7421 case NEON::BI__builtin_neon_vld3_lane_v:
7422 case NEON::BI__builtin_neon_vld3q_lane_v: {
7423 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
7424 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
7425 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
7426 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7427 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7428 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
7430 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld3_lane");
7431 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7432 }
7433 case NEON::BI__builtin_neon_vld4_lane_v:
7434 case NEON::BI__builtin_neon_vld4q_lane_v: {
7435 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
7436 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
7437 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
7438 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7439 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7440 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
7441 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
7443 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld4_lane");
7444 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7445 }
7446 case NEON::BI__builtin_neon_vst2_v:
7447 case NEON::BI__builtin_neon_vst2q_v: {
7448 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
7449 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
7450 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
7451 Ops, "");
7452 }
7453 case NEON::BI__builtin_neon_vst2_lane_v:
7454 case NEON::BI__builtin_neon_vst2q_lane_v: {
7455 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
7457 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
7458 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
7459 Ops, "");
7460 }
7461 case NEON::BI__builtin_neon_vst3_v:
7462 case NEON::BI__builtin_neon_vst3q_v: {
7463 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
7464 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
7465 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
7466 Ops, "");
7467 }
7468 case NEON::BI__builtin_neon_vst3_lane_v:
7469 case NEON::BI__builtin_neon_vst3q_lane_v: {
7470 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
7472 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
7473 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
7474 Ops, "");
7475 }
7476 case NEON::BI__builtin_neon_vst4_v:
7477 case NEON::BI__builtin_neon_vst4q_v: {
7478 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
7479 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
7480 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
7481 Ops, "");
7482 }
7483 case NEON::BI__builtin_neon_vst4_lane_v:
7484 case NEON::BI__builtin_neon_vst4q_lane_v: {
7485 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
7487 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
7488 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
7489 Ops, "");
7490 }
7491 case NEON::BI__builtin_neon_vtrn_v:
7492 case NEON::BI__builtin_neon_vtrnq_v: {
7493 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7494 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7495 Value *SV = nullptr;
7496
7497 for (unsigned vi = 0; vi != 2; ++vi) {
7499 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
7500 Indices.push_back(i+vi);
7501 Indices.push_back(i+e+vi);
7502 }
7503 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7504 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
7505 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
7506 }
7507 return SV;
7508 }
7509 case NEON::BI__builtin_neon_vuzp_v:
7510 case NEON::BI__builtin_neon_vuzpq_v: {
7511 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7512 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7513 Value *SV = nullptr;
7514
7515 for (unsigned vi = 0; vi != 2; ++vi) {
7517 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
7518 Indices.push_back(2*i+vi);
7519
7520 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7521 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
7522 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
7523 }
7524 return SV;
7525 }
7526 case NEON::BI__builtin_neon_vzip_v:
7527 case NEON::BI__builtin_neon_vzipq_v: {
7528 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7529 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7530 Value *SV = nullptr;
7531
7532 for (unsigned vi = 0; vi != 2; ++vi) {
7534 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
7535 Indices.push_back((i + vi*e) >> 1);
7536 Indices.push_back(((i + vi*e) >> 1)+e);
7537 }
7538 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7539 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
7540 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
7541 }
7542 return SV;
7543 }
7544 case NEON::BI__builtin_neon_vqtbl1q_v: {
7545 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
7546 Ops, "vtbl1");
7547 }
7548 case NEON::BI__builtin_neon_vqtbl2q_v: {
7549 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
7550 Ops, "vtbl2");
7551 }
7552 case NEON::BI__builtin_neon_vqtbl3q_v: {
7553 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
7554 Ops, "vtbl3");
7555 }
7556 case NEON::BI__builtin_neon_vqtbl4q_v: {
7557 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
7558 Ops, "vtbl4");
7559 }
7560 case NEON::BI__builtin_neon_vqtbx1q_v: {
7561 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
7562 Ops, "vtbx1");
7563 }
7564 case NEON::BI__builtin_neon_vqtbx2q_v: {
7565 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
7566 Ops, "vtbx2");
7567 }
7568 case NEON::BI__builtin_neon_vqtbx3q_v: {
7569 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
7570 Ops, "vtbx3");
7571 }
7572 case NEON::BI__builtin_neon_vqtbx4q_v: {
7573 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
7574 Ops, "vtbx4");
7575 }
7576 case NEON::BI__builtin_neon_vsqadd_v:
7577 case NEON::BI__builtin_neon_vsqaddq_v: {
7578 Int = Intrinsic::aarch64_neon_usqadd;
7579 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd");
7580 }
7581 case NEON::BI__builtin_neon_vuqadd_v:
7582 case NEON::BI__builtin_neon_vuqaddq_v: {
7583 Int = Intrinsic::aarch64_neon_suqadd;
7584 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd");
7585 }
7586
7587 case NEON::BI__builtin_neon_vluti2_laneq_mf8:
7588 case NEON::BI__builtin_neon_vluti2_laneq_bf16:
7589 case NEON::BI__builtin_neon_vluti2_laneq_f16:
7590 case NEON::BI__builtin_neon_vluti2_laneq_p16:
7591 case NEON::BI__builtin_neon_vluti2_laneq_p8:
7592 case NEON::BI__builtin_neon_vluti2_laneq_s16:
7593 case NEON::BI__builtin_neon_vluti2_laneq_s8:
7594 case NEON::BI__builtin_neon_vluti2_laneq_u16:
7595 case NEON::BI__builtin_neon_vluti2_laneq_u8: {
7596 Int = Intrinsic::aarch64_neon_vluti2_laneq;
7597 llvm::Type *Tys[2];
7598 Tys[0] = Ty;
7600 false));
7601 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vluti2_laneq");
7602 }
7603 case NEON::BI__builtin_neon_vluti2q_laneq_mf8:
7604 case NEON::BI__builtin_neon_vluti2q_laneq_bf16:
7605 case NEON::BI__builtin_neon_vluti2q_laneq_f16:
7606 case NEON::BI__builtin_neon_vluti2q_laneq_p16:
7607 case NEON::BI__builtin_neon_vluti2q_laneq_p8:
7608 case NEON::BI__builtin_neon_vluti2q_laneq_s16:
7609 case NEON::BI__builtin_neon_vluti2q_laneq_s8:
7610 case NEON::BI__builtin_neon_vluti2q_laneq_u16:
7611 case NEON::BI__builtin_neon_vluti2q_laneq_u8: {
7612 Int = Intrinsic::aarch64_neon_vluti2_laneq;
7613 llvm::Type *Tys[2];
7614 Tys[0] = Ty;
7616 true));
7617 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vluti2_laneq");
7618 }
7619 case NEON::BI__builtin_neon_vluti2_lane_mf8:
7620 case NEON::BI__builtin_neon_vluti2_lane_bf16:
7621 case NEON::BI__builtin_neon_vluti2_lane_f16:
7622 case NEON::BI__builtin_neon_vluti2_lane_p16:
7623 case NEON::BI__builtin_neon_vluti2_lane_p8:
7624 case NEON::BI__builtin_neon_vluti2_lane_s16:
7625 case NEON::BI__builtin_neon_vluti2_lane_s8:
7626 case NEON::BI__builtin_neon_vluti2_lane_u16:
7627 case NEON::BI__builtin_neon_vluti2_lane_u8: {
7628 Int = Intrinsic::aarch64_neon_vluti2_lane;
7629 llvm::Type *Tys[2];
7630 Tys[0] = Ty;
7632 false));
7633 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vluti2_lane");
7634 }
7635 case NEON::BI__builtin_neon_vluti2q_lane_mf8:
7636 case NEON::BI__builtin_neon_vluti2q_lane_bf16:
7637 case NEON::BI__builtin_neon_vluti2q_lane_f16:
7638 case NEON::BI__builtin_neon_vluti2q_lane_p16:
7639 case NEON::BI__builtin_neon_vluti2q_lane_p8:
7640 case NEON::BI__builtin_neon_vluti2q_lane_s16:
7641 case NEON::BI__builtin_neon_vluti2q_lane_s8:
7642 case NEON::BI__builtin_neon_vluti2q_lane_u16:
7643 case NEON::BI__builtin_neon_vluti2q_lane_u8: {
7644 Int = Intrinsic::aarch64_neon_vluti2_lane;
7645 llvm::Type *Tys[2];
7646 Tys[0] = Ty;
7648 true));
7649 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vluti2_lane");
7650 }
7651 case NEON::BI__builtin_neon_vluti4q_lane_mf8:
7652 case NEON::BI__builtin_neon_vluti4q_lane_p8:
7653 case NEON::BI__builtin_neon_vluti4q_lane_s8:
7654 case NEON::BI__builtin_neon_vluti4q_lane_u8: {
7655 Int = Intrinsic::aarch64_neon_vluti4q_lane;
7656 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_lane");
7657 }
7658 case NEON::BI__builtin_neon_vluti4q_laneq_mf8:
7659 case NEON::BI__builtin_neon_vluti4q_laneq_p8:
7660 case NEON::BI__builtin_neon_vluti4q_laneq_s8:
7661 case NEON::BI__builtin_neon_vluti4q_laneq_u8: {
7662 Int = Intrinsic::aarch64_neon_vluti4q_laneq;
7663 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_laneq");
7664 }
7665 case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2:
7666 case NEON::BI__builtin_neon_vluti4q_lane_f16_x2:
7667 case NEON::BI__builtin_neon_vluti4q_lane_p16_x2:
7668 case NEON::BI__builtin_neon_vluti4q_lane_s16_x2:
7669 case NEON::BI__builtin_neon_vluti4q_lane_u16_x2: {
7670 Int = Intrinsic::aarch64_neon_vluti4q_lane_x2;
7671 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_lane_x2");
7672 }
7673 case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2:
7674 case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2:
7675 case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2:
7676 case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2:
7677 case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2: {
7678 Int = Intrinsic::aarch64_neon_vluti4q_laneq_x2;
7679 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vluti4q_laneq_x2");
7680 }
7681 case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm:
7683 {llvm::FixedVectorType::get(HalfTy, 8),
7684 llvm::FixedVectorType::get(Int8Ty, 16)},
7685 Ops, E, "fmmla");
7686 case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm:
7688 {llvm::FixedVectorType::get(FloatTy, 4),
7689 llvm::FixedVectorType::get(Int8Ty, 16)},
7690 Ops, E, "fmmla");
7691 case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm:
7692 ExtractLow = true;
7693 [[fallthrough]];
7694 case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm:
7695 case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm:
7697 llvm::FixedVectorType::get(BFloatTy, 8),
7698 Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt1");
7699 case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm:
7700 ExtractLow = true;
7701 [[fallthrough]];
7702 case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm:
7703 case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm:
7705 llvm::FixedVectorType::get(BFloatTy, 8),
7706 Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt2");
7707 case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm:
7708 ExtractLow = true;
7709 [[fallthrough]];
7710 case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm:
7711 case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm:
7713 llvm::FixedVectorType::get(HalfTy, 8),
7714 Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt1");
7715 case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm:
7716 ExtractLow = true;
7717 [[fallthrough]];
7718 case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm:
7719 case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm:
7721 llvm::FixedVectorType::get(HalfTy, 8),
7722 Ops[0]->getType(), ExtractLow, Ops, E, "vbfcvt2");
7723 case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm:
7725 llvm::FixedVectorType::get(Int8Ty, 8),
7726 Ops[0]->getType(), false, Ops, E, "vfcvtn");
7727 case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm:
7729 llvm::FixedVectorType::get(Int8Ty, 8),
7730 llvm::FixedVectorType::get(HalfTy, 4), false, Ops,
7731 E, "vfcvtn");
7732 case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm:
7734 llvm::FixedVectorType::get(Int8Ty, 16),
7735 llvm::FixedVectorType::get(HalfTy, 8), false, Ops,
7736 E, "vfcvtn");
7737 case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm: {
7738 llvm::Type *Ty = llvm::FixedVectorType::get(Int8Ty, 16);
7739 Ops[0] = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
7740 uint64_t(0));
7742 Ops[1]->getType(), false, Ops, E, "vfcvtn2");
7743 }
7744
7745 case NEON::BI__builtin_neon_vdot_f16_mf8_fpm:
7746 case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm:
7748 Ops, E, "fdot2");
7749 case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm:
7750 case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm:
7751 ExtendLaneArg = true;
7752 [[fallthrough]];
7753 case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm:
7754 case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm:
7756 ExtendLaneArg, HalfTy, Ops, E, "fdot2_lane");
7757 case NEON::BI__builtin_neon_vdot_f32_mf8_fpm:
7758 case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm:
7760 FloatTy, Ops, E, "fdot4");
7761 case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm:
7762 case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm:
7763 ExtendLaneArg = true;
7764 [[fallthrough]];
7765 case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm:
7766 case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm:
7768 ExtendLaneArg, FloatTy, Ops, E, "fdot4_lane");
7769
7770 case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm:
7771 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalb,
7772 {llvm::FixedVectorType::get(HalfTy, 8)}, Ops, E,
7773 "vmlal");
7774 case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm:
7775 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalt,
7776 {llvm::FixedVectorType::get(HalfTy, 8)}, Ops, E,
7777 "vmlal");
7778 case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm:
7779 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlallbb,
7780 {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
7781 "vmlall");
7782 case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm:
7783 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlallbt,
7784 {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
7785 "vmlall");
7786 case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm:
7787 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalltb,
7788 {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
7789 "vmlall");
7790 case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm:
7791 return EmitFP8NeonCall(Intrinsic::aarch64_neon_fp8_fmlalltt,
7792 {llvm::FixedVectorType::get(FloatTy, 4)}, Ops, E,
7793 "vmlall");
7794 case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm:
7795 ExtendLaneArg = true;
7796 [[fallthrough]];
7797 case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm:
7799 ExtendLaneArg, HalfTy, Ops, E, "vmlal_lane");
7800 case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm:
7801 ExtendLaneArg = true;
7802 [[fallthrough]];
7803 case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm:
7805 ExtendLaneArg, HalfTy, Ops, E, "vmlal_lane");
7806 case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm:
7807 ExtendLaneArg = true;
7808 [[fallthrough]];
7809 case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm:
7811 ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
7812 case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm:
7813 ExtendLaneArg = true;
7814 [[fallthrough]];
7815 case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm:
7817 ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
7818 case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm:
7819 ExtendLaneArg = true;
7820 [[fallthrough]];
7821 case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm:
7823 ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
7824 case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm:
7825 ExtendLaneArg = true;
7826 [[fallthrough]];
7827 case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm:
7829 ExtendLaneArg, FloatTy, Ops, E, "vmlall_lane");
7830 case NEON::BI__builtin_neon_vamin_f16:
7831 case NEON::BI__builtin_neon_vaminq_f16:
7832 case NEON::BI__builtin_neon_vamin_f32:
7833 case NEON::BI__builtin_neon_vaminq_f32:
7834 case NEON::BI__builtin_neon_vaminq_f64: {
7835 Int = Intrinsic::aarch64_neon_famin;
7836 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "famin");
7837 }
7838 case NEON::BI__builtin_neon_vamax_f16:
7839 case NEON::BI__builtin_neon_vamaxq_f16:
7840 case NEON::BI__builtin_neon_vamax_f32:
7841 case NEON::BI__builtin_neon_vamaxq_f32:
7842 case NEON::BI__builtin_neon_vamaxq_f64: {
7843 Int = Intrinsic::aarch64_neon_famax;
7844 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "famax");
7845 }
7846 case NEON::BI__builtin_neon_vscale_f16:
7847 case NEON::BI__builtin_neon_vscaleq_f16:
7848 case NEON::BI__builtin_neon_vscale_f32:
7849 case NEON::BI__builtin_neon_vscaleq_f32:
7850 case NEON::BI__builtin_neon_vscaleq_f64: {
7851 Int = Intrinsic::aarch64_neon_fp8_fscale;
7852 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fscale");
7853 }
7854 }
7855}
7856
7859 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
7860 BuiltinID == BPF::BI__builtin_btf_type_id ||
7861 BuiltinID == BPF::BI__builtin_preserve_type_info ||
7862 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
7863 "unexpected BPF builtin");
7864
7865
7866
7867
7868 static uint32_t BuiltinSeqNum;
7869
7870 switch (BuiltinID) {
7871 default:
7872 llvm_unreachable("Unexpected BPF builtin");
7873 case BPF::BI__builtin_preserve_field_info: {
7876
7879 "using __builtin_preserve_field_info() without -g");
7882 }
7883
7884
7890
7892 Value *InfoKind = ConstantInt::get(Int64Ty, C->getSExtValue());
7893
7894
7895 llvm::Function *FnGetFieldInfo = Intrinsic::getOrInsertDeclaration(
7896 &CGM.getModule(), Intrinsic::bpf_preserve_field_info,
7897 {FieldAddr->getType()});
7898 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
7899 }
7900 case BPF::BI__builtin_btf_type_id:
7901 case BPF::BI__builtin_preserve_type_info: {
7903 CGM.Error(E->getExprLoc(), "using builtin function without -g");
7904 return nullptr;
7905 }
7906
7910
7912 Value *FlagValue = ConstantInt::get(Int64Ty, Flag->getSExtValue());
7913 Value *SeqNumVal = ConstantInt::get(Int32Ty, BuiltinSeqNum++);
7914
7915 llvm::Function *FnDecl;
7916 if (BuiltinID == BPF::BI__builtin_btf_type_id)
7917 FnDecl = Intrinsic::getOrInsertDeclaration(
7918 &CGM.getModule(), Intrinsic::bpf_btf_type_id, {});
7919 else
7920 FnDecl = Intrinsic::getOrInsertDeclaration(
7921 &CGM.getModule(), Intrinsic::bpf_preserve_type_info, {});
7922 CallInst *Fn = Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
7923 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
7924 return Fn;
7925 }
7926 case BPF::BI__builtin_preserve_enum_value: {
7928 CGM.Error(E->getExprLoc(), "using builtin function without -g");
7929 return nullptr;
7930 }
7931
7935
7936
7941
7942 auto InitVal = Enumerator->getInitVal();
7943 std::string InitValStr;
7944 if (InitVal.isNegative() || InitVal > uint64_t(INT64_MAX))
7945 InitValStr = std::to_string(InitVal.getSExtValue());
7946 else
7947 InitValStr = std::to_string(InitVal.getZExtValue());
7948 std::string EnumStr = Enumerator->getNameAsString() + ":" + InitValStr;
7949 Value *EnumStrVal = Builder.CreateGlobalString(EnumStr);
7950
7952 Value *FlagValue = ConstantInt::get(Int64Ty, Flag->getSExtValue());
7953 Value *SeqNumVal = ConstantInt::get(Int32Ty, BuiltinSeqNum++);
7954
7955 llvm::Function *IntrinsicFn = Intrinsic::getOrInsertDeclaration(
7956 &CGM.getModule(), Intrinsic::bpf_preserve_enum_value, {});
7957 CallInst *Fn =
7958 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
7959 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
7960 return Fn;
7961 }
7962 }
7963}
7964
7967 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
7968 "Not a power-of-two sized vector!");
7969 bool AllConstants = true;
7970 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
7972
7973
7974 if (AllConstants) {
7976 for (llvm::Value *Op : Ops)
7978 return llvm::ConstantVector::get(CstOps);
7979 }
7980
7981
7983 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
7984
7985 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
7987
7989}
7990
7991Value *CodeGenFunction::EmitAArch64CpuInit() {
7992 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
7993 llvm::FunctionCallee Func =
7997 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
7998 return Builder.CreateCall(Func);
7999}
8000
8001Value *CodeGenFunction::EmitAArch64CpuSupports(const CallExpr *E) {
8005 ArgStr.split(Features, "+");
8006 for (auto &Feature : Features) {
8008 if (!llvm::AArch64::parseFMVExtension(Feature))
8009 return Builder.getFalse();
8010 if (Feature != "default")
8011 Features.push_back(Feature);
8012 }
8013 return EmitAArch64CpuSupports(Features);
8014}
8015
8016llvm::Value *
8017CodeGenFunction::EmitAArch64CpuSupports(ArrayRef FeaturesStrs) {
8018 llvm::APInt FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs);
8020 if (FeaturesMask != 0) {
8021
8022
8023
8024
8025 llvm::Type *STy = llvm::StructType::get(Int64Ty);
8026 llvm::Constant *AArch64CPUFeatures =
8027 CGM.CreateRuntimeVariable(STy, "__aarch64_cpu_features");
8029 llvm::Value *CpuFeatures = Builder.CreateGEP(
8030 STy, AArch64CPUFeatures,
8031 {ConstantInt::get(Int32Ty, 0), ConstantInt::get(Int32Ty, 0)});
8034 Value *Mask = Builder.getInt(FeaturesMask.trunc(64));
8035 Value *Bitset = Builder.CreateAnd(Features, Mask);
8036 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
8038 }
8040}
Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
Definition ARM.cpp:2597
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
Definition ARM.cpp:3437
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
Definition ARM.cpp:588
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
Definition ARM.cpp:401
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
Definition ARM.cpp:3407
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
Definition ARM.cpp:3400
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
Definition ARM.cpp:4481
static const ARMVectorIntrinsicInfo AArch64SMEIntrinsicMap[]
Definition ARM.cpp:1649
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
Definition ARM.cpp:3659
static void swapCommutativeSMEOperands(unsigned BuiltinID, SmallVectorImpl< Value * > &Ops)
Definition ARM.cpp:4948
static bool AArch64SISDIntrinsicsProvenSorted
Definition ARM.cpp:1661
static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
Definition ARM.cpp:1631
static llvm::Value * ARMMVECreateFPToSI(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *Ty)
Definition ARM.cpp:3531
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
Definition ARM.cpp:2665
static llvm::Value * ARMMVECreateFPToUI(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *Ty)
Definition ARM.cpp:3539
static llvm::Value * ARMMVECreateSIToFP(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *Ty)
Definition ARM.cpp:3515
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
Definition ARM.cpp:584
static bool AArch64SVEIntrinsicsProvenSorted
Definition ARM.cpp:1662
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
Definition ARM.cpp:1666
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
Definition ARM.cpp:4487
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
Definition ARM.cpp:3396
static bool AArch64SMEIntrinsicsProvenSorted
Definition ARM.cpp:1663
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
Definition ARM.cpp:3474
constexpr unsigned SVEBitsPerBlock
Definition ARM.cpp:3946
static const std::pair< unsigned, unsigned > NEONEquivalentIntrinsicMap[]
Definition ARM.cpp:1473
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasFastHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
Definition ARM.cpp:359
#define NEONMAP0(NameBase)
Definition ARM.cpp:581
Value * readX18AsPtr(CodeGenFunction &CGF)
Helper for the read/write/add/inc X18 builtins: read the X18 register and return it as an i8 pointer.
Definition ARM.cpp:5041
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
Definition ARM.cpp:3501
static std::optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
Definition ARM.cpp:31
static std::optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
Definition ARM.cpp:190
static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
Definition ARM.cpp:594
static llvm::Value * ARMMVECreateUIToFP(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *Ty)
Definition ARM.cpp:3523
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
Definition ARM.cpp:3463
static llvm::ScalableVectorType * getSVEVectorForElementType(llvm::Type *EltTy)
Definition ARM.cpp:3948
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
Definition ARM.cpp:3489
SpecialRegisterAccessKind
Definition ARM.cpp:2588
@ VolatileRead
Definition ARM.cpp:2590
@ NormalRead
Definition ARM.cpp:2589
@ Write
Definition ARM.cpp:2591
@ UnsignedAlts
Definition ARM.cpp:551
@ Vectorize1ArgType
Definition ARM.cpp:556
@ FpCmpzModifiers
Definition ARM.cpp:560
@ Use64BitVectors
Definition ARM.cpp:553
@ VectorizeArgTypes
Definition ARM.cpp:548
@ VectorRetGetArgs01
Definition ARM.cpp:558
@ InventFloatType
Definition ARM.cpp:550
@ AddRetType
Definition ARM.cpp:543
@ Add2ArgTypes
Definition ARM.cpp:545
@ VectorizeRetType
Definition ARM.cpp:547
@ VectorRet
Definition ARM.cpp:557
@ Add1ArgType
Definition ARM.cpp:544
@ Use128BitVectors
Definition ARM.cpp:554
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
Definition ARM.cpp:3429
static bool NEONSIMDIntrinsicsProvenSorted
Definition ARM.cpp:1658
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
Definition ARM.cpp:913
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
Definition ARM.cpp:1724
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
Definition ARM.cpp:342
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
Definition ARM.cpp:2515
static bool AArch64SIMDIntrinsicsProvenSorted
Definition ARM.cpp:1660
static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
Definition ARM.cpp:1188
TokenType getType() const
Returns the token's type, e.g.
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
HLSLResourceBindingAttr::RegisterType RegisterType
Enumerates target-specific builtins in their own namespaces within namespace clang.
__device__ __2f16 float __ockl_bool s
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
FunctionDecl * getDirectCallee()
If the callee is a FunctionDecl, return it. Otherwise return null.
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
static CharUnits One()
One - Construct a CharUnits quantity of one.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
llvm::PointerType * getType() const
Return the type of the pointer value.
Address getAddress() const
llvm::DIType * getOrCreateStandaloneType(QualType Ty, SourceLocation Loc)
Emit standalone debug info for a type.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
Definition ARM.cpp:3955
llvm::Value * EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy, SmallVectorImpl< llvm::Value * > &Ops, const CallExpr *E, const char *name)
Definition ARM.cpp:474
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
Definition ARM.cpp:7966
llvm::Value * EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx, const CallExpr *E)
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:4150
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
Definition ARM.cpp:4258
llvm::Value * EmitFP8NeonCall(unsigned IID, ArrayRef< llvm::Type * > Tys, SmallVectorImpl< llvm::Value * > &O, const CallExpr *E, const char *name)
Definition ARM.cpp:448
llvm::Type * ConvertType(QualType T)
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:4117
llvm::Value * EmitSMEReadWrite(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:4405
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
Definition ARM.cpp:3814
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:4060
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
Definition ARM.cpp:4315
llvm::Value * EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:4972
void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, SVETypeFlags TypeFlags)
Definition ARM.cpp:4542
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:4006
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
Definition ARM.cpp:1685
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
Definition ARM.cpp:3830
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
Definition ARM.cpp:1786
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
const TargetInfo & getTarget() const
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:4585
llvm::Value * EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0, llvm::Type *Ty1, bool Extract, SmallVectorImpl< llvm::Value * > &Ops, const CallExpr *E, const char *name)
Definition ARM.cpp:495
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
Definition ARM.cpp:2710
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
Definition ARM.cpp:3903
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
Definition ARM.cpp:7857
llvm::Value * EmitSMELdrStr(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:4427
llvm::Value * EmitSVETupleCreate(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
Definition ARM.cpp:4530
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
Definition ARM.cpp:4213
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
Definition ARM.cpp:3547
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
Definition ARM.cpp:511
CGDebugInfo * getDebugInfo()
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
Definition ARM.cpp:4494
const TargetCodeGenInfo & getTargetHooks() const
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
Definition ARM.cpp:489
bool IsInPreservedAIRegion
True if CodeGen currently emits code inside presereved access index region.
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
Definition ARM.cpp:5052
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
llvm::Value * EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg, llvm::Type *RetTy, SmallVectorImpl< llvm::Value * > &Ops, const CallExpr *E, const char *name)
Definition ARM.cpp:458
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
Definition ARM.cpp:3802
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
Definition ARM.cpp:3648
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Pred, const llvm::Twine &Name="")
Definition ARM.cpp:2488
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
Definition ARM.cpp:4231
llvm::Value * EmitSVEPredicateTupleCast(llvm::Value *PredTuple, llvm::StructType *Ty)
Definition ARM.cpp:3990
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
Definition ARM.cpp:4238
llvm::Value * EmitSMEZero(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:4417
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:4165
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
llvm::Value * EmitSMELd1St1(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Definition ARM.cpp:4370
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
llvm::Value * EmitSVEAllTruePred(const SVETypeFlags &TypeFlags)
Definition ARM.cpp:3940
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
Definition ARM.cpp:4458
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
llvm::LLVMContext & getLLVMContext()
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
Definition ARM.cpp:3868
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
Definition ARM.cpp:427
llvm::Value * EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, ArrayRef< llvm::Value * > Ops)
Definition ARM.cpp:4519
This class organizes the cross-function state that is used while generating LLVM code.
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
llvm::LLVMContext & getLLVMContext()
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys={})
llvm::Value * getRawBitFieldPointer(CodeGenFunction &CGF) const
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
const T & getABIInfo() const
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Represents a function declaration or definition.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
Flags to identify the types for overloaded Neon builtins.
EltType getEltType() const
PointerType - C99 6.7.5.1 - Pointer Declarators.
QualType getPointeeType() const
A (possibly-)qualified type.
The collection of all-type qualifiers we support.
Flags to identify the types for overloaded SVE builtins.
bool isZExtReturn() const
bool isReverseUSDOT() const
bool isOverloadNone() const
MemEltType getMemEltType() const
bool isGatherLoad() const
EltType getEltType() const
bool isOverloadFirstandLast() const
bool isOverloadDefault() const
bool isOverloadWhileRW() const
bool isReverseMergeAnyAccOp() const
bool isReductionQV() const
bool isInsertOp1SVALL() const
bool isAppendSVALL() const
bool isReverseMergeAnyBinOp() const
bool isStructStore() const
bool isTupleCreate() const
bool isGatherPrefetch() const
bool hasSplatOperand() const
MergeType getMergeType() const
bool isByteIndexed() const
bool isStructLoad() const
bool isOverloadWhileOrMultiVecCvt() const
unsigned getSplatOperand() const
bool isScatterStore() const
bool isReverseCompare() const
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
virtual bool hasFastHalfType() const
Determine whether the target has fast native support for operations on half types.
The base class of the type hierarchy.
const T * castAs() const
Member-template castAs.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
The JSON file list parser is used to communicate input to InstallAPI.
bool isa(CodeGen::Address addr)
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
@ Result
The result type of a method or function.
const FunctionProtoType * T
U cast(CodeGen::Address addr)
@ Enumerator
Enumerator value with fixed underlying type.
Diagnostic wrappers for TextAPI types for error reporting.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::Type * HalfTy
half, bfloat, float, double
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntTy
int
llvm::IntegerType * Int16Ty
llvm::PointerType * Int8PtrTy
llvm::PointerType * DefaultPtrTy
llvm::PointerType * AllocaInt8PtrTy
EvalResult is a struct with detailed info about an evaluated expression.