LLVM: lib/IR/AutoUpgrade.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
30#include "llvm/IR/IntrinsicsAArch64.h"
31#include "llvm/IR/IntrinsicsARM.h"
32#include "llvm/IR/IntrinsicsNVPTX.h"
33#include "llvm/IR/IntrinsicsRISCV.h"
34#include "llvm/IR/IntrinsicsWebAssembly.h"
35#include "llvm/IR/IntrinsicsX86.h"
47#include
48#include
49
50using namespace llvm;
51
54 cl::desc("Disable autoupgrade of debug info"));
55
57
58
59
62
63
64 Type *Arg0Type = F->getFunctionType()->getParamType(0);
66 return false;
67
68
71 return true;
72}
73
74
75
78
79 Type *LastArgType = F->getFunctionType()->getParamType(
80 F->getFunctionType()->getNumParams() - 1);
82 return false;
83
84
87 return true;
88}
89
90
91
94
95 if (F->getReturnType()->isVectorTy())
96 return false;
97
100 return true;
101}
102
105 if (F->getReturnType()->getScalarType()->isBFloatTy())
106 return false;
107
110 return true;
111}
112
115 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
116 return false;
117
120 return true;
121}
122
124
125
126
127
128
129 if (Name.consume_front("avx."))
130 return (Name.starts_with("blend.p") ||
131 Name == "cvt.ps2.pd.256" ||
132 Name == "cvtdq2.pd.256" ||
133 Name == "cvtdq2.ps.256" ||
134 Name.starts_with("movnt.") ||
135 Name.starts_with("sqrt.p") ||
136 Name.starts_with("storeu.") ||
137 Name.starts_with("vbroadcast.s") ||
138 Name.starts_with("vbroadcastf128") ||
139 Name.starts_with("vextractf128.") ||
140 Name.starts_with("vinsertf128.") ||
141 Name.starts_with("vperm2f128.") ||
142 Name.starts_with("vpermil."));
143
144 if (Name.consume_front("avx2."))
145 return (Name == "movntdqa" ||
146 Name.starts_with("pabs.") ||
147 Name.starts_with("padds.") ||
148 Name.starts_with("paddus.") ||
149 Name.starts_with("pblendd.") ||
150 Name == "pblendw" ||
151 Name.starts_with("pbroadcast") ||
152 Name.starts_with("pcmpeq.") ||
153 Name.starts_with("pcmpgt.") ||
154 Name.starts_with("pmax") ||
155 Name.starts_with("pmin") ||
156 Name.starts_with("pmovsx") ||
157 Name.starts_with("pmovzx") ||
158 Name == "pmul.dq" ||
159 Name == "pmulu.dq" ||
160 Name.starts_with("psll.dq") ||
161 Name.starts_with("psrl.dq") ||
162 Name.starts_with("psubs.") ||
163 Name.starts_with("psubus.") ||
164 Name.starts_with("vbroadcast") ||
165 Name == "vbroadcasti128" ||
166 Name == "vextracti128" ||
167 Name == "vinserti128" ||
168 Name == "vperm2i128");
169
170 if (Name.consume_front("avx512.")) {
171 if (Name.consume_front("mask."))
172
173 return (Name.starts_with("add.p") ||
174 Name.starts_with("and.") ||
175 Name.starts_with("andn.") ||
176 Name.starts_with("broadcast.s") ||
177 Name.starts_with("broadcastf32x4.") ||
178 Name.starts_with("broadcastf32x8.") ||
179 Name.starts_with("broadcastf64x2.") ||
180 Name.starts_with("broadcastf64x4.") ||
181 Name.starts_with("broadcasti32x4.") ||
182 Name.starts_with("broadcasti32x8.") ||
183 Name.starts_with("broadcasti64x2.") ||
184 Name.starts_with("broadcasti64x4.") ||
185 Name.starts_with("cmp.b") ||
186 Name.starts_with("cmp.d") ||
187 Name.starts_with("cmp.q") ||
188 Name.starts_with("cmp.w") ||
189 Name.starts_with("compress.b") ||
190 Name.starts_with("compress.d") ||
191 Name.starts_with("compress.p") ||
192 Name.starts_with("compress.q") ||
193 Name.starts_with("compress.store.") ||
194 Name.starts_with("compress.w") ||
195 Name.starts_with("conflict.") ||
196 Name.starts_with("cvtdq2pd.") ||
197 Name.starts_with("cvtdq2ps.") ||
198 Name == "cvtpd2dq.256" ||
199 Name == "cvtpd2ps.256" ||
200 Name == "cvtps2pd.128" ||
201 Name == "cvtps2pd.256" ||
202 Name.starts_with("cvtqq2pd.") ||
203 Name == "cvtqq2ps.256" ||
204 Name == "cvtqq2ps.512" ||
205 Name == "cvttpd2dq.256" ||
206 Name == "cvttps2dq.128" ||
207 Name == "cvttps2dq.256" ||
208 Name.starts_with("cvtudq2pd.") ||
209 Name.starts_with("cvtudq2ps.") ||
210 Name.starts_with("cvtuqq2pd.") ||
211 Name == "cvtuqq2ps.256" ||
212 Name == "cvtuqq2ps.512" ||
213 Name.starts_with("dbpsadbw.") ||
214 Name.starts_with("div.p") ||
215 Name.starts_with("expand.b") ||
216 Name.starts_with("expand.d") ||
217 Name.starts_with("expand.load.") ||
218 Name.starts_with("expand.p") ||
219 Name.starts_with("expand.q") ||
220 Name.starts_with("expand.w") ||
221 Name.starts_with("fpclass.p") ||
222 Name.starts_with("insert") ||
223 Name.starts_with("load.") ||
224 Name.starts_with("loadu.") ||
225 Name.starts_with("lzcnt.") ||
226 Name.starts_with("max.p") ||
227 Name.starts_with("min.p") ||
228 Name.starts_with("movddup") ||
229 Name.starts_with("move.s") ||
230 Name.starts_with("movshdup") ||
231 Name.starts_with("movsldup") ||
232 Name.starts_with("mul.p") ||
233 Name.starts_with("or.") ||
234 Name.starts_with("pabs.") ||
235 Name.starts_with("packssdw.") ||
236 Name.starts_with("packsswb.") ||
237 Name.starts_with("packusdw.") ||
238 Name.starts_with("packuswb.") ||
239 Name.starts_with("padd.") ||
240 Name.starts_with("padds.") ||
241 Name.starts_with("paddus.") ||
242 Name.starts_with("palignr.") ||
243 Name.starts_with("pand.") ||
244 Name.starts_with("pandn.") ||
245 Name.starts_with("pavg") ||
246 Name.starts_with("pbroadcast") ||
247 Name.starts_with("pcmpeq.") ||
248 Name.starts_with("pcmpgt.") ||
249 Name.starts_with("perm.df.") ||
250 Name.starts_with("perm.di.") ||
251 Name.starts_with("permvar.") ||
252 Name.starts_with("pmaddubs.w.") ||
253 Name.starts_with("pmaddw.d.") ||
254 Name.starts_with("pmax") ||
255 Name.starts_with("pmin") ||
256 Name == "pmov.qd.256" ||
257 Name == "pmov.qd.512" ||
258 Name == "pmov.wb.256" ||
259 Name == "pmov.wb.512" ||
260 Name.starts_with("pmovsx") ||
261 Name.starts_with("pmovzx") ||
262 Name.starts_with("pmul.dq.") ||
263 Name.starts_with("pmul.hr.sw.") ||
264 Name.starts_with("pmulh.w.") ||
265 Name.starts_with("pmulhu.w.") ||
266 Name.starts_with("pmull.") ||
267 Name.starts_with("pmultishift.qb.") ||
268 Name.starts_with("pmulu.dq.") ||
269 Name.starts_with("por.") ||
270 Name.starts_with("prol.") ||
271 Name.starts_with("prolv.") ||
272 Name.starts_with("pror.") ||
273 Name.starts_with("prorv.") ||
274 Name.starts_with("pshuf.b.") ||
275 Name.starts_with("pshuf.d.") ||
276 Name.starts_with("pshufh.w.") ||
277 Name.starts_with("pshufl.w.") ||
278 Name.starts_with("psll.d") ||
279 Name.starts_with("psll.q") ||
280 Name.starts_with("psll.w") ||
281 Name.starts_with("pslli") ||
282 Name.starts_with("psllv") ||
283 Name.starts_with("psra.d") ||
284 Name.starts_with("psra.q") ||
285 Name.starts_with("psra.w") ||
286 Name.starts_with("psrai") ||
287 Name.starts_with("psrav") ||
288 Name.starts_with("psrl.d") ||
289 Name.starts_with("psrl.q") ||
290 Name.starts_with("psrl.w") ||
291 Name.starts_with("psrli") ||
292 Name.starts_with("psrlv") ||
293 Name.starts_with("psub.") ||
294 Name.starts_with("psubs.") ||
295 Name.starts_with("psubus.") ||
296 Name.starts_with("pternlog.") ||
297 Name.starts_with("punpckh") ||
298 Name.starts_with("punpckl") ||
299 Name.starts_with("pxor.") ||
300 Name.starts_with("shuf.f") ||
301 Name.starts_with("shuf.i") ||
302 Name.starts_with("shuf.p") ||
303 Name.starts_with("sqrt.p") ||
304 Name.starts_with("store.b.") ||
305 Name.starts_with("store.d.") ||
306 Name.starts_with("store.p") ||
307 Name.starts_with("store.q.") ||
308 Name.starts_with("store.w.") ||
309 Name == "store.ss" ||
310 Name.starts_with("storeu.") ||
311 Name.starts_with("sub.p") ||
312 Name.starts_with("ucmp.") ||
313 Name.starts_with("unpckh.") ||
314 Name.starts_with("unpckl.") ||
315 Name.starts_with("valign.") ||
316 Name == "vcvtph2ps.128" ||
317 Name == "vcvtph2ps.256" ||
318 Name.starts_with("vextract") ||
319 Name.starts_with("vfmadd.") ||
320 Name.starts_with("vfmaddsub.") ||
321 Name.starts_with("vfnmadd.") ||
322 Name.starts_with("vfnmsub.") ||
323 Name.starts_with("vpdpbusd.") ||
324 Name.starts_with("vpdpbusds.") ||
325 Name.starts_with("vpdpwssd.") ||
326 Name.starts_with("vpdpwssds.") ||
327 Name.starts_with("vpermi2var.") ||
328 Name.starts_with("vpermil.p") ||
329 Name.starts_with("vpermilvar.") ||
330 Name.starts_with("vpermt2var.") ||
331 Name.starts_with("vpmadd52") ||
332 Name.starts_with("vpshld.") ||
333 Name.starts_with("vpshldv.") ||
334 Name.starts_with("vpshrd.") ||
335 Name.starts_with("vpshrdv.") ||
336 Name.starts_with("vpshufbitqmb.") ||
337 Name.starts_with("xor."));
338
339 if (Name.consume_front("mask3."))
340
341 return (Name.starts_with("vfmadd.") ||
342 Name.starts_with("vfmaddsub.") ||
343 Name.starts_with("vfmsub.") ||
344 Name.starts_with("vfmsubadd.") ||
345 Name.starts_with("vfnmsub."));
346
347 if (Name.consume_front("maskz."))
348
349 return (Name.starts_with("pternlog.") ||
350 Name.starts_with("vfmadd.") ||
351 Name.starts_with("vfmaddsub.") ||
352 Name.starts_with("vpdpbusd.") ||
353 Name.starts_with("vpdpbusds.") ||
354 Name.starts_with("vpdpwssd.") ||
355 Name.starts_with("vpdpwssds.") ||
356 Name.starts_with("vpermt2var.") ||
357 Name.starts_with("vpmadd52") ||
358 Name.starts_with("vpshldv.") ||
359 Name.starts_with("vpshrdv."));
360
361
362 return (Name == "movntdqa" ||
363 Name == "pmul.dq.512" ||
364 Name == "pmulu.dq.512" ||
365 Name.starts_with("broadcastm") ||
366 Name.starts_with("cmp.p") ||
367 Name.starts_with("cvtb2mask.") ||
368 Name.starts_with("cvtd2mask.") ||
369 Name.starts_with("cvtmask2") ||
370 Name.starts_with("cvtq2mask.") ||
371 Name == "cvtusi2sd" ||
372 Name.starts_with("cvtw2mask.") ||
373 Name == "kand.w" ||
374 Name == "kandn.w" ||
375 Name == "knot.w" ||
376 Name == "kor.w" ||
377 Name == "kortestc.w" ||
378 Name == "kortestz.w" ||
379 Name.starts_with("kunpck") ||
380 Name == "kxnor.w" ||
381 Name == "kxor.w" ||
382 Name.starts_with("padds.") ||
383 Name.starts_with("pbroadcast") ||
384 Name.starts_with("prol") ||
385 Name.starts_with("pror") ||
386 Name.starts_with("psll.dq") ||
387 Name.starts_with("psrl.dq") ||
388 Name.starts_with("psubs.") ||
389 Name.starts_with("ptestm") ||
390 Name.starts_with("ptestnm") ||
391 Name.starts_with("storent.") ||
392 Name.starts_with("vbroadcast.s") ||
393 Name.starts_with("vpshld.") ||
394 Name.starts_with("vpshrd."));
395 }
396
397 if (Name.consume_front("fma."))
398 return (Name.starts_with("vfmadd.") ||
399 Name.starts_with("vfmsub.") ||
400 Name.starts_with("vfmsubadd.") ||
401 Name.starts_with("vfnmadd.") ||
402 Name.starts_with("vfnmsub."));
403
404 if (Name.consume_front("fma4."))
405 return Name.starts_with("vfmadd.s");
406
407 if (Name.consume_front("sse."))
408 return (Name == "add.ss" ||
409 Name == "cvtsi2ss" ||
410 Name == "cvtsi642ss" ||
411 Name == "div.ss" ||
412 Name == "mul.ss" ||
413 Name.starts_with("sqrt.p") ||
414 Name == "sqrt.ss" ||
415 Name.starts_with("storeu.") ||
416 Name == "sub.ss");
417
418 if (Name.consume_front("sse2."))
419 return (Name == "add.sd" ||
420 Name == "cvtdq2pd" ||
421 Name == "cvtdq2ps" ||
422 Name == "cvtps2pd" ||
423 Name == "cvtsi2sd" ||
424 Name == "cvtsi642sd" ||
425 Name == "cvtss2sd" ||
426 Name == "div.sd" ||
427 Name == "mul.sd" ||
428 Name.starts_with("padds.") ||
429 Name.starts_with("paddus.") ||
430 Name.starts_with("pcmpeq.") ||
431 Name.starts_with("pcmpgt.") ||
432 Name == "pmaxs.w" ||
433 Name == "pmaxu.b" ||
434 Name == "pmins.w" ||
435 Name == "pminu.b" ||
436 Name == "pmulu.dq" ||
437 Name.starts_with("pshuf") ||
438 Name.starts_with("psll.dq") ||
439 Name.starts_with("psrl.dq") ||
440 Name.starts_with("psubs.") ||
441 Name.starts_with("psubus.") ||
442 Name.starts_with("sqrt.p") ||
443 Name == "sqrt.sd" ||
444 Name == "storel.dq" ||
445 Name.starts_with("storeu.") ||
446 Name == "sub.sd");
447
448 if (Name.consume_front("sse41."))
449 return (Name.starts_with("blendp") ||
450 Name == "movntdqa" ||
451 Name == "pblendw" ||
452 Name == "pmaxsb" ||
453 Name == "pmaxsd" ||
454 Name == "pmaxud" ||
455 Name == "pmaxuw" ||
456 Name == "pminsb" ||
457 Name == "pminsd" ||
458 Name == "pminud" ||
459 Name == "pminuw" ||
460 Name.starts_with("pmovsx") ||
461 Name.starts_with("pmovzx") ||
462 Name == "pmuldq");
463
464 if (Name.consume_front("sse42."))
465 return Name == "crc32.64.8";
466
467 if (Name.consume_front("sse4a."))
468 return Name.starts_with("movnt.");
469
470 if (Name.consume_front("ssse3."))
471 return (Name == "pabs.b.128" ||
472 Name == "pabs.d.128" ||
473 Name == "pabs.w.128");
474
475 if (Name.consume_front("xop."))
476 return (Name == "vpcmov" ||
477 Name == "vpcmov.256" ||
478 Name.starts_with("vpcom") ||
479 Name.starts_with("vprot"));
480
481 return (Name == "addcarry.u32" ||
482 Name == "addcarry.u64" ||
483 Name == "addcarryx.u32" ||
484 Name == "addcarryx.u64" ||
485 Name == "subborrow.u32" ||
486 Name == "subborrow.u64" ||
487 Name.starts_with("vcvtph2ps."));
488}
489
492
493 if (.consume_front("x86."))
494 return false;
495
497 NewFn = nullptr;
498 return true;
499 }
500
501 if (Name == "rdtscp") {
502
503 if (F->getFunctionType()->getNumParams() == 0)
504 return false;
505
508 Intrinsic::x86_rdtscp);
509 return true;
510 }
511
513
514
515 if (Name.consume_front("sse41.ptest")) {
517 .Case("c", Intrinsic::x86_sse41_ptestc)
518 .Case("z", Intrinsic::x86_sse41_ptestz)
519 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
523
524 return false;
525 }
526
527
528
529
530
532 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
533 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
534 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
535 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
536 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
537 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
541
542 if (Name.consume_front("avx512.mask.cmp.")) {
543
545 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
546 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
547 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
548 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
549 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
550 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
554 return false;
555 }
556
557 if (Name.consume_front("avx512bf16.")) {
558
560 .Case("cvtne2ps2bf16.128",
561 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
562 .Case("cvtne2ps2bf16.256",
563 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
564 .Case("cvtne2ps2bf16.512",
565 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
566 .Case("mask.cvtneps2bf16.128",
567 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
568 .Case("cvtneps2bf16.256",
569 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
570 .Case("cvtneps2bf16.512",
571 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
575
576
578 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
579 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
580 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
584 return false;
585 }
586
587 if (Name.consume_front("xop.")) {
589 if (Name.starts_with("vpermil2")) {
590
591
592 auto Idx = F->getFunctionType()->getParamType(2);
593 if (Idx->isFPOrFPVectorTy()) {
594 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
595 unsigned EltSize = Idx->getScalarSizeInBits();
596 if (EltSize == 64 && IdxSize == 128)
597 ID = Intrinsic::x86_xop_vpermil2pd;
598 else if (EltSize == 32 && IdxSize == 128)
599 ID = Intrinsic::x86_xop_vpermil2ps;
600 else if (EltSize == 64 && IdxSize == 256)
601 ID = Intrinsic::x86_xop_vpermil2pd_256;
602 else
603 ID = Intrinsic::x86_xop_vpermil2ps_256;
604 }
605 } else if (F->arg_size() == 2)
606
608 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
609 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
611
615 return true;
616 }
617 return false;
618 }
619
620 if (Name == "seh.recoverfp") {
622 Intrinsic::eh_recoverfp);
623 return true;
624 }
625
626 return false;
627}
628
629
630
634 if (Name.starts_with("rbit")) {
635
637 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
638 return true;
639 }
640
641 if (Name == "thread.pointer") {
642
644 Intrinsic::thread_pointer);
645 return true;
646 }
647
648 bool Neon = Name.consume_front("neon.");
649 if (Neon) {
650
651
652
653 if (Name.consume_front("bfdot.")) {
654
657 .Cases("v2f32.v8i8", "v4f32.v16i8",
662 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
663 assert((OperandWidth == 64 || OperandWidth == 128) &&
664 "Unexpected operand width");
665 LLVMContext &Ctx = F->getParent()->getContext();
666 std::array<Type *, 2> Tys{
667 {F->getReturnType(),
670 return true;
671 }
672 return false;
673 }
674
675
676
677 if (Name.consume_front("bfm")) {
678
679 if (Name.consume_back(".v4f32.v16i8")) {
680
684 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
686 .Case("lalb",
687 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
688 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
689 .Case("lalt",
690 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
691 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
695 return true;
696 }
697 return false;
698 }
699 return false;
700 }
701
702 }
703
704
705 if (IsArm) {
706
707 if (Neon) {
708
710 .StartsWith("vclz.", Intrinsic::ctlz)
711 .StartsWith("vcnt.", Intrinsic::ctpop)
712 .StartsWith("vqadds.", Intrinsic::sadd_sat)
713 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
714 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
715 .StartsWith("vqsubu.", Intrinsic::usub_sat)
719 F->arg_begin()->getType());
720 return true;
721 }
722
723 if (Name.consume_front("vst")) {
724
725 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
729 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
730 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
731
733 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
734 Intrinsic::arm_neon_vst4lane};
735
736 auto fArgs = F->getFunctionType()->params();
737 Type *Tys[] = {fArgs[0], fArgs[1]};
740 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
741 else
743 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
744 return true;
745 }
746 return false;
747 }
748
749 return false;
750 }
751
752 if (Name.consume_front("mve.")) {
753
754 if (Name == "vctp64") {
755 if (cast(F->getReturnType())->getNumElements() == 4) {
756
757
759 return true;
760 }
761 return false;
762 }
763
764
765 if (Name.consume_back(".v4i1")) {
766
767 if (Name.consume_back(".predicated.v2i64.v4i32"))
768
769 return Name == "mull.int" || Name == "vqdmull";
770
771 if (Name.consume_back(".v2i64")) {
772
773 bool IsGather = Name.consume_front("vldr.gather.");
774 if (IsGather || Name.consume_front("vstr.scatter.")) {
775 if (Name.consume_front("base.")) {
776
777 Name.consume_front("wb.");
778
779
780 return Name == "predicated.v2i64";
781 }
782
783 if (Name.consume_front("offset.predicated."))
784 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
785 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
786
787
788 return false;
789 }
790
791 return false;
792 }
793 return false;
794 }
795 return false;
796 }
797
798 if (Name.consume_front("cde.vcx")) {
799
800 if (Name.consume_back(".predicated.v2i64.v4i1"))
801
802 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
803 Name == "3q" || Name == "3qa";
804
805 return false;
806 }
807 } else {
808
809 if (Neon) {
810
812 .StartsWith("frintn", Intrinsic::roundeven)
813 .StartsWith("rbit", Intrinsic::bitreverse)
817 F->arg_begin()->getType());
818 return true;
819 }
820
821 if (Name.starts_with("addp")) {
822
823 if (F->arg_size() != 2)
824 return false;
825 VectorType *Ty = dyn_cast(F->getReturnType());
826 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
828 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
829 return true;
830 }
831 }
832
833
834 if (Name.starts_with("bfcvt")) {
835 NewFn = nullptr;
836 return true;
837 }
838
839 return false;
840 }
841 if (Name.consume_front("sve.")) {
842
843 if (Name.consume_front("bf")) {
844 if (Name.consume_back(".lane")) {
845
848 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
849 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
850 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
854 return true;
855 }
856 return false;
857 }
858 return false;
859 }
860
861
862 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
863 NewFn = nullptr;
864 return true;
865 }
866
867 if (Name.consume_front("addqv")) {
868
869 if (->getReturnType()->isFPOrFPVectorTy())
870 return false;
871
872 auto Args = F->getFunctionType()->params();
873 Type *Tys[] = {F->getReturnType(), Args[1]};
875 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
876 return true;
877 }
878
879 if (Name.consume_front("ld")) {
880
881 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
883 Type *ScalarTy =
884 cast(F->getReturnType())->getElementType();
886 cast(F->arg_begin()->getType())->getElementCount();
887 Type *Ty = VectorType::get(ScalarTy, EC);
889 Intrinsic::aarch64_sve_ld2_sret,
890 Intrinsic::aarch64_sve_ld3_sret,
891 Intrinsic::aarch64_sve_ld4_sret,
892 };
894 LoadIDs[Name[0] - '2'], Ty);
895 return true;
896 }
897 return false;
898 }
899
900 if (Name.consume_front("tuple.")) {
901
902 if (Name.starts_with("get")) {
903
904 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
906 F->getParent(), Intrinsic::vector_extract, Tys);
907 return true;
908 }
909
910 if (Name.starts_with("set")) {
911
912 auto Args = F->getFunctionType()->params();
913 Type *Tys[] = {Args[0], Args[2], Args[1]};
915 F->getParent(), Intrinsic::vector_insert, Tys);
916 return true;
917 }
918
919 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
920 if (CreateTupleRegex.match(Name)) {
921
922 auto Args = F->getFunctionType()->params();
923 Type *Tys[] = {F->getReturnType(), Args[1]};
925 F->getParent(), Intrinsic::vector_insert, Tys);
926 return true;
927 }
928 return false;
929 }
930 return false;
931 }
932 }
933 return false;
934}
935
937 if (Name.consume_front("abs."))
939 .Case("bf16", Intrinsic::nvvm_abs_bf16)
940 .Case("bf16x2", Intrinsic::nvvm_abs_bf16x2)
942
943 if (Name.consume_front("fma.rn."))
945 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
946 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
947 .Case("ftz.bf16", Intrinsic::nvvm_fma_rn_ftz_bf16)
948 .Case("ftz.bf16x2", Intrinsic::nvvm_fma_rn_ftz_bf16x2)
949 .Case("ftz.relu.bf16", Intrinsic::nvvm_fma_rn_ftz_relu_bf16)
950 .Case("ftz.relu.bf16x2", Intrinsic::nvvm_fma_rn_ftz_relu_bf16x2)
951 .Case("ftz.sat.bf16", Intrinsic::nvvm_fma_rn_ftz_sat_bf16)
952 .Case("ftz.sat.bf16x2", Intrinsic::nvvm_fma_rn_ftz_sat_bf16x2)
953 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
954 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
955 .Case("sat.bf16", Intrinsic::nvvm_fma_rn_sat_bf16)
956 .Case("sat.bf16x2", Intrinsic::nvvm_fma_rn_sat_bf16x2)
958
959 if (Name.consume_front("fmax."))
961 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
962 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
963 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
964 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
965 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
966 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
967 .Case("ftz.nan.xorsign.abs.bf16",
968 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
969 .Case("ftz.nan.xorsign.abs.bf16x2",
970 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
971 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
972 .Case("ftz.xorsign.abs.bf16x2",
973 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
974 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
975 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
976 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
977 .Case("nan.xorsign.abs.bf16x2",
978 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
979 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
980 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
982
983 if (Name.consume_front("fmin."))
985 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
986 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
987 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
988 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
989 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
990 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
991 .Case("ftz.nan.xorsign.abs.bf16",
992 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
993 .Case("ftz.nan.xorsign.abs.bf16x2",
994 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
995 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
996 .Case("ftz.xorsign.abs.bf16x2",
997 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
998 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
999 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1000 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1001 .Case("nan.xorsign.abs.bf16x2",
1002 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1003 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1004 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1006
1007 if (Name.consume_front("neg."))
1009 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1010 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1012
1014}
1015
1017 bool CanUpgradeDebugIntrinsicsToRecords) {
1018 assert(F && "Illegal to upgrade a non-existent Function.");
1019
1021
1022
1023 if (.consume_front("llvm.") || Name.empty())
1024 return false;
1025
1026 switch (Name[0]) {
1027 default: break;
1028 case 'a': {
1029 bool IsArm = Name.consume_front("arm.");
1030 if (IsArm || Name.consume_front("aarch64.")) {
1032 return true;
1033 break;
1034 }
1035
1036 if (Name.consume_front("amdgcn.")) {
1037 if (Name == "alignbit") {
1038
1040 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1041 return true;
1042 }
1043
1044 if (Name.consume_front("atomic.")) {
1045 if (Name.starts_with("inc") || Name.starts_with("dec")) {
1046
1047
1048 NewFn = nullptr;
1049 return true;
1050 }
1051 break;
1052 }
1053
1054 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1055 Name.consume_front("flat.atomic.")) {
1056 if (Name.starts_with("fadd") ||
1057
1058 (Name.starts_with("fmin") && .starts_with("fmin.num")) ||
1059 (Name.starts_with("fmax") && .starts_with("fmax.num"))) {
1060
1061
1062 NewFn = nullptr;
1063 return true;
1064 }
1065 }
1066
1067 if (Name.starts_with("ldexp.")) {
1068
1070 F->getParent(), Intrinsic::ldexp,
1071 {F->getReturnType(), F->getArg(1)->getType()});
1072 return true;
1073 }
1074 break;
1075 }
1076
1077 break;
1078 }
1079 case 'c': {
1080 if (F->arg_size() == 1) {
1082 .StartsWith("ctlz.", Intrinsic::ctlz)
1083 .StartsWith("cttz.", Intrinsic::cttz)
1088 F->arg_begin()->getType());
1089 return true;
1090 }
1091 }
1092
1093 if (F->arg_size() == 2 && Name == "coro.end") {
1096 Intrinsic::coro_end);
1097 return true;
1098 }
1099
1100 break;
1101 }
1102 case 'd':
1103 if (Name.consume_front("dbg.")) {
1104
1105 if (CanUpgradeDebugIntrinsicsToRecords &&
1106 F->getParent()->IsNewDbgInfoFormat) {
1107 if (Name == "addr" || Name == "value" || Name == "assign" ||
1108 Name == "declare" || Name == "label") {
1109
1110 NewFn = nullptr;
1111
1112 return true;
1113 }
1114 }
1115
1116
1117 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1120 Intrinsic::dbg_value);
1121 return true;
1122 }
1123 break;
1124 }
1125 break;
1126 case 'e':
1127 if (Name.consume_front("experimental.vector.")) {
1130
1131
1133 .StartsWith("extract.", Intrinsic::vector_extract)
1134 .StartsWith("insert.", Intrinsic::vector_insert)
1135 .StartsWith("splice.", Intrinsic::vector_splice)
1136 .StartsWith("reverse.", Intrinsic::vector_reverse)
1137 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1138 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1141 const auto *FT = F->getFunctionType();
1143 if (ID == Intrinsic::vector_extract ||
1144 ID == Intrinsic::vector_interleave2)
1145
1146 Tys.push_back(FT->getReturnType());
1147 if (ID != Intrinsic::vector_interleave2)
1148 Tys.push_back(FT->getParamType(0));
1149 if (ID == Intrinsic::vector_insert)
1150
1151 Tys.push_back(FT->getParamType(1));
1154 return true;
1155 }
1156
1157 if (Name.consume_front("reduce.")) {
1159 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1162 .Case("add", Intrinsic::vector_reduce_add)
1163 .Case("mul", Intrinsic::vector_reduce_mul)
1164 .Case("and", Intrinsic::vector_reduce_and)
1165 .Case("or", Intrinsic::vector_reduce_or)
1166 .Case("xor", Intrinsic::vector_reduce_xor)
1167 .Case("smax", Intrinsic::vector_reduce_smax)
1168 .Case("smin", Intrinsic::vector_reduce_smin)
1169 .Case("umax", Intrinsic::vector_reduce_umax)
1170 .Case("umin", Intrinsic::vector_reduce_umin)
1171 .Case("fmax", Intrinsic::vector_reduce_fmax)
1172 .Case("fmin", Intrinsic::vector_reduce_fmin)
1174
1175 bool V2 = false;
1177 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1179 V2 = true;
1182 .Case("fadd", Intrinsic::vector_reduce_fadd)
1183 .Case("fmul", Intrinsic::vector_reduce_fmul)
1185 }
1188 auto Args = F->getFunctionType()->params();
1190 {Args[V2 ? 1 : 0]});
1191 return true;
1192 }
1193 break;
1194 }
1195 break;
1196 }
1197 if (Name.consume_front("experimental.stepvector.")) {
1201 F->getParent(), ID, F->getFunctionType()->getReturnType());
1202 return true;
1203 }
1204 break;
1205 case 'f':
1206 if (Name.starts_with("flt.rounds")) {
1209 Intrinsic::get_rounding);
1210 return true;
1211 }
1212 break;
1213 case 'i':
1214 if (Name.starts_with("invariant.group.barrier")) {
1215
1216 auto Args = F->getFunctionType()->params();
1217 Type* ObjectPtr[1] = {Args[0]};
1220 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1221 return true;
1222 }
1223 break;
1224 case 'm': {
1225
1226
1227
1229 .StartsWith("memcpy.", Intrinsic::memcpy)
1230 .StartsWith("memmove.", Intrinsic::memmove)
1232 if (F->arg_size() == 5) {
1234
1236 F->getFunctionType()->params().slice(0, 3);
1237 NewFn =
1239 return true;
1240 }
1241 }
1242 if (Name.starts_with("memset.") && F->arg_size() == 5) {
1244
1245 const auto *FT = F->getFunctionType();
1246 Type *ParamTypes[2] = {
1247 FT->getParamType(0),
1248 FT->getParamType(2)
1249 };
1251 Intrinsic::memset, ParamTypes);
1252 return true;
1253 }
1254 break;
1255 }
1256 case 'n': {
1257 if (Name.consume_front("nvvm.")) {
1258
1259 if (F->arg_size() == 1) {
1262 .Cases("brev32", "brev64", Intrinsic::bitreverse)
1263 .Case("clz.i", Intrinsic::ctlz)
1264 .Case("popc.i", Intrinsic::ctpop)
1268 {F->getReturnType()});
1269 return true;
1270 }
1271 }
1272
1273
1274 if (->getReturnType()->getScalarType()->isBFloatTy()) {
1277 NewFn = nullptr;
1278 return true;
1279 }
1280 }
1281
1282
1283
1284
1285
1286 bool Expand = false;
1287 if (Name.consume_front("abs."))
1288
1289 Expand = Name == "i" || Name == "ll";
1290 else if (Name == "clz.ll" || Name == "popc.ll" || Name == "h2f")
1291 Expand = true;
1292 else if (Name.consume_front("max.") || Name.consume_front("min."))
1293
1294 Expand = Name == "s" || Name == "i" || Name == "ll" || Name == "us" ||
1295 Name == "ui" || Name == "ull";
1296 else if (Name.consume_front("atomic.load.add."))
1297
1298 Expand = Name.starts_with("f32.p") || Name.starts_with("f64.p");
1299 else if (Name.consume_front("bitcast."))
1300
1301 Expand =
1302 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
1303 else if (Name.consume_front("rotate."))
1304
1305 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
1306 else if (Name.consume_front("ptr.gen.to."))
1307
1308 Expand = Name.starts_with("local") || Name.starts_with("shared") ||
1309 Name.starts_with("global") || Name.starts_with("constant");
1310 else if (Name.consume_front("ptr."))
1311
1312 Expand =
1313 (Name.consume_front("local") || Name.consume_front("shared") ||
1314 Name.consume_front("global") || Name.consume_front("constant")) &&
1315 Name.starts_with(".to.gen");
1316 else if (Name.consume_front("ldg.global."))
1317
1318 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
1319 Name.starts_with("p."));
1320 else
1321 Expand = false;
1322
1323 if (Expand) {
1324 NewFn = nullptr;
1325 return true;
1326 }
1327 break;
1328 }
1329 break;
1330 }
1331 case 'o':
1332
1333
1334 if (Name.starts_with("objectsize.")) {
1335 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
1336 if (F->arg_size() == 2 || F->arg_size() == 3 ||
1337 F->getName() !=
1341 Intrinsic::objectsize, Tys);
1342 return true;
1343 }
1344 }
1345 break;
1346
1347 case 'p':
1348 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
1351 F->getParent(), Intrinsic::ptr_annotation,
1352 {F->arg_begin()->getType(), F->getArg(1)->getType()});
1353 return true;
1354 }
1355 break;
1356
1357 case 'r': {
1358 if (Name.consume_front("riscv.")) {
1361 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
1362 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
1363 .Case("aes32esi", Intrinsic::riscv_aes32esi)
1364 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
1367 if (->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
1370 return true;
1371 }
1372 break;
1373 }
1374
1376 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
1377 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
1380 if (->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
1381 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1384 return true;
1385 }
1386 break;
1387 }
1388
1390 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
1391 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
1392 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
1393 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
1394 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
1395 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
1398 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1401 return true;
1402 }
1403 break;
1404 }
1405 break;
1406 }
1407 } break;
1408
1409 case 's':
1410 if (Name == "stackprotectorcheck") {
1411 NewFn = nullptr;
1412 return true;
1413 }
1414 break;
1415
1416 case 'v': {
1417 if (Name == "var.annotation" && F->arg_size() == 4) {
1420 F->getParent(), Intrinsic::var_annotation,
1421 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
1422 return true;
1423 }
1424 break;
1425 }
1426
1427 case 'w':
1428 if (Name.consume_front("wasm.")) {
1431 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
1432 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
1433 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
1438 F->getReturnType());
1439 return true;
1440 }
1441
1442 if (Name.consume_front("dot.i8x16.i7x16.")) {
1444 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
1445 .Case("add.signed",
1446 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
1451 return true;
1452 }
1453 break;
1454 }
1455 break;
1456 }
1457 break;
1458
1459 case 'x':
1461 return true;
1462 }
1463
1464 auto *ST = dyn_cast(F->getReturnType());
1465 if (ST && (->isLiteral() || ST->isPacked()) &&
1467
1468
1469
1470
1474 auto *FT = F->getFunctionType();
1476 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
1477 std::string Name = F->getName().str();
1479 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
1481
1482
1485 return true;
1486 }
1487 }
1488
1489
1491 if (Result != std::nullopt) {
1493 return true;
1494 }
1495
1496
1497
1498
1499
1500 return false;
1501}
1502
1504 bool CanUpgradeDebugIntrinsicsToRecords) {
1505 NewFn = nullptr;
1506 bool Upgraded =
1508 assert(F != NewFn && "Intrinsic function upgraded to the same function");
1509
1510
1511 if (NewFn)
1512 F = NewFn;
1515 return Upgraded;
1516}
1517
1519 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
1520 GV->getName() == "llvm.global_dtors")) ||
1522 return nullptr;
1524 if (!ATy)
1525 return nullptr;
1528 return nullptr;
1529
1535 unsigned N = Init->getNumOperands();
1536 std::vector<Constant *> NewCtors(N);
1537 for (unsigned i = 0; i != N; ++i) {
1538 auto Ctor = cast(Init->getOperand(i));
1540 Ctor->getAggregateElement(1),
1542 }
1544
1546 NewInit, GV->getName());
1547}
1548
1549
1550
1552 unsigned Shift) {
1553 auto *ResultTy = cast(Op->getType());
1554 unsigned NumElts = ResultTy->getNumElements() * 8;
1555
1556
1559
1560
1562
1563
1564
1565 if (Shift < 16) {
1566 int Idxs[64];
1567
1568 for (unsigned l = 0; l != NumElts; l += 16)
1569 for (unsigned i = 0; i != 16; ++i) {
1570 unsigned Idx = NumElts + i - Shift;
1571 if (Idx < NumElts)
1572 Idx -= NumElts - 16;
1573 Idxs[l + i] = Idx + l;
1574 }
1575
1577 }
1578
1579
1580 return Builder.CreateBitCast(Res, ResultTy, "cast");
1581}
1582
1583
1584
1586 unsigned Shift) {
1587 auto *ResultTy = cast(Op->getType());
1588 unsigned NumElts = ResultTy->getNumElements() * 8;
1589
1590
1593
1594
1596
1597
1598
1599 if (Shift < 16) {
1600 int Idxs[64];
1601
1602 for (unsigned l = 0; l != NumElts; l += 16)
1603 for (unsigned i = 0; i != 16; ++i) {
1604 unsigned Idx = i + Shift;
1605 if (Idx >= 16)
1606 Idx += NumElts - 16;
1607 Idxs[l + i] = Idx + l;
1608 }
1609
1611 }
1612
1613
1614 return Builder.CreateBitCast(Res, ResultTy, "cast");
1615}
1616
1618 unsigned NumElts) {
1623
1624
1625
1626 if (NumElts <= 4) {
1627 int Indices[4];
1628 for (unsigned i = 0; i != NumElts; ++i)
1629 Indices[i] = i;
1631 "extract");
1632 }
1633
1634 return Mask;
1635}
1636
1639
1640 if (const auto *C = dyn_cast(Mask))
1641 if (C->isAllOnesValue())
1642 return Op0;
1643
1645 cast(Op0->getType())->getNumElements());
1646 return Builder.CreateSelect(Mask, Op0, Op1);
1647}
1648
1651
1652 if (const auto *C = dyn_cast(Mask))
1653 if (C->isAllOnesValue())
1654 return Op0;
1655
1657 Mask->getType()->getIntegerBitWidth());
1660 return Builder.CreateSelect(Mask, Op0, Op1);
1661}
1662
1663
1664
1665
1669 bool IsVALIGN) {
1670 unsigned ShiftVal = castllvm::ConstantInt(Shift)->getZExtValue();
1671
1672 unsigned NumElts = cast(Op0->getType())->getNumElements();
1673 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
1674 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
1676
1677
1678 if (IsVALIGN)
1679 ShiftVal &= (NumElts - 1);
1680
1681
1682
1683 if (ShiftVal >= 32)
1685
1686
1687
1688 if (ShiftVal > 16) {
1689 ShiftVal -= 16;
1690 Op1 = Op0;
1692 }
1693
1694 int Indices[64];
1695
1696 for (unsigned l = 0; l < NumElts; l += 16) {
1697 for (unsigned i = 0; i != 16; ++i) {
1698 unsigned Idx = ShiftVal + i;
1699 if (!IsVALIGN && Idx >= 16)
1700 Idx += NumElts - 16;
1701 Indices[l + i] = Idx + l;
1702 }
1703 }
1704
1706 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
1707
1709}
1710
1712 bool ZeroMask, bool IndexForm) {
1718 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
1719 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1720 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
1721 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1722 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
1723 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1724 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
1725 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1726 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1727 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1728 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1729 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1730 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1731 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1732 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1733 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1734 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1735 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1736 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1737 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1738 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1739 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1740 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1741 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1742 else if (VecWidth == 128 && EltWidth == 16)
1743 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1744 else if (VecWidth == 256 && EltWidth == 16)
1745 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1746 else if (VecWidth == 512 && EltWidth == 16)
1747 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1748 else if (VecWidth == 128 && EltWidth == 8)
1749 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1750 else if (VecWidth == 256 && EltWidth == 8)
1751 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1752 else if (VecWidth == 512 && EltWidth == 8)
1753 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1754 else
1756
1759
1760
1761 if (!IndexForm)
1763
1767 Ty);
1769}
1770
1777
1778 if (CI.arg_size() == 4) {
1781 Res = emitX86Select(Builder, Mask, Res, VecSrc);
1782 }
1783 return Res;
1784}
1785
1787 bool IsRotateRight) {
1791
1792
1793
1794
1795 if (Amt->getType() != Ty) {
1796 unsigned NumElts = cast(Ty)->getNumElements();
1799 }
1800
1801 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1803
1804 if (CI.arg_size() == 4) {
1807 Res = emitX86Select(Builder, Mask, Res, VecSrc);
1808 }
1809 return Res;
1810}
1811
1813 bool IsSigned) {
1817
1819 switch (Imm) {
1820 case 0x0:
1821 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
1822 break;
1823 case 0x1:
1824 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
1825 break;
1826 case 0x2:
1827 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
1828 break;
1829 case 0x3:
1830 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
1831 break;
1832 case 0x4:
1833 Pred = ICmpInst::ICMP_EQ;
1834 break;
1835 case 0x5:
1836 Pred = ICmpInst::ICMP_NE;
1837 break;
1838 case 0x6:
1840 case 0x7:
1842 default:
1844 }
1845
1848 return Ext;
1849}
1850
1852 bool IsShiftRight, bool ZeroMask) {
1857
1858 if (IsShiftRight)
1860
1861
1862
1863
1864 if (Amt->getType() != Ty) {
1865 unsigned NumElts = cast(Ty)->getNumElements();
1868 }
1869
1870 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1872
1873 unsigned NumArgs = CI.arg_size();
1874 if (NumArgs >= 4) {
1879 Res = emitX86Select(Builder, Mask, Res, VecSrc);
1880 }
1881 return Res;
1882}
1883
1886 const Align Alignment =
1888 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
1890
1891
1892 if (const auto *C = dyn_cast(Mask))
1893 if (C->isAllOnesValue())
1895
1896
1897 unsigned NumElts = cast(Data->getType())->getNumElements();
1900}
1901
1905 const Align Alignment =
1909 8)
1911
1912
1913 if (const auto *C = dyn_cast(Mask))
1914 if (C->isAllOnesValue())
1916
1917
1918 unsigned NumElts = cast(ValTy)->getNumElements();
1921}
1922
1927 {Op0, Builder.getInt1(false)});
1930 return Res;
1931}
1932
1935
1936
1939
1940 if (IsSigned) {
1941
1942 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
1947 } else {
1948
1949 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
1952 }
1953
1955
1958
1959 return Res;
1960}
1961
1962
1965 unsigned NumElts = cast(Vec->getType())->getNumElements();
1966 if (Mask) {
1967 const auto *C = dyn_cast(Mask);
1968 if ( ||
->isAllOnesValue())
1970 }
1971
1972 if (NumElts < 8) {
1973 int Indices[8];
1974 for (unsigned i = 0; i != NumElts; ++i)
1975 Indices[i] = i;
1976 for (unsigned i = NumElts; i != 8; ++i)
1977 Indices[i] = NumElts + i % NumElts;
1980 Indices);
1981 }
1983}
1984
1988 unsigned NumElts = cast(Op0->getType())->getNumElements();
1989
1991 if (CC == 3) {
1994 } else if (CC == 7) {
1997 } else {
1999 switch (CC) {
2001 case 0: Pred = ICmpInst::ICMP_EQ; break;
2002 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2003 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2004 case 4: Pred = ICmpInst::ICMP_NE; break;
2005 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2006 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2007 }
2009 }
2010
2012
2014}
2015
2016
2022}
2023
2029
2036}
2037
2041 unsigned NumElts = cast(CI.getType())->getNumElements();
2043 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
2044}
2045
2046
2050
2054 if (Name.starts_with("max.p")) {
2055 if (VecWidth == 128 && EltWidth == 32)
2056 IID = Intrinsic::x86_sse_max_ps;
2057 else if (VecWidth == 128 && EltWidth == 64)
2058 IID = Intrinsic::x86_sse2_max_pd;
2059 else if (VecWidth == 256 && EltWidth == 32)
2060 IID = Intrinsic::x86_avx_max_ps_256;
2061 else if (VecWidth == 256 && EltWidth == 64)
2062 IID = Intrinsic::x86_avx_max_pd_256;
2063 else
2065 } else if (Name.starts_with("min.p")) {
2066 if (VecWidth == 128 && EltWidth == 32)
2067 IID = Intrinsic::x86_sse_min_ps;
2068 else if (VecWidth == 128 && EltWidth == 64)
2069 IID = Intrinsic::x86_sse2_min_pd;
2070 else if (VecWidth == 256 && EltWidth == 32)
2071 IID = Intrinsic::x86_avx_min_ps_256;
2072 else if (VecWidth == 256 && EltWidth == 64)
2073 IID = Intrinsic::x86_avx_min_pd_256;
2074 else
2076 } else if (Name.starts_with("pshuf.b.")) {
2077 if (VecWidth == 128)
2078 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2079 else if (VecWidth == 256)
2080 IID = Intrinsic::x86_avx2_pshuf_b;
2081 else if (VecWidth == 512)
2082 IID = Intrinsic::x86_avx512_pshuf_b_512;
2083 else
2085 } else if (Name.starts_with("pmul.hr.sw.")) {
2086 if (VecWidth == 128)
2087 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2088 else if (VecWidth == 256)
2089 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2090 else if (VecWidth == 512)
2091 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2092 else
2094 } else if (Name.starts_with("pmulh.w.")) {
2095 if (VecWidth == 128)
2096 IID = Intrinsic::x86_sse2_pmulh_w;
2097 else if (VecWidth == 256)
2098 IID = Intrinsic::x86_avx2_pmulh_w;
2099 else if (VecWidth == 512)
2100 IID = Intrinsic::x86_avx512_pmulh_w_512;
2101 else
2103 } else if (Name.starts_with("pmulhu.w.")) {
2104 if (VecWidth == 128)
2105 IID = Intrinsic::x86_sse2_pmulhu_w;
2106 else if (VecWidth == 256)
2107 IID = Intrinsic::x86_avx2_pmulhu_w;
2108 else if (VecWidth == 512)
2109 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2110 else
2112 } else if (Name.starts_with("pmaddw.d.")) {
2113 if (VecWidth == 128)
2114 IID = Intrinsic::x86_sse2_pmadd_wd;
2115 else if (VecWidth == 256)
2116 IID = Intrinsic::x86_avx2_pmadd_wd;
2117 else if (VecWidth == 512)
2118 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2119 else
2121 } else if (Name.starts_with("pmaddubs.w.")) {
2122 if (VecWidth == 128)
2123 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2124 else if (VecWidth == 256)
2125 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2126 else if (VecWidth == 512)
2127 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2128 else
2130 } else if (Name.starts_with("packsswb.")) {
2131 if (VecWidth == 128)
2132 IID = Intrinsic::x86_sse2_packsswb_128;
2133 else if (VecWidth == 256)
2134 IID = Intrinsic::x86_avx2_packsswb;
2135 else if (VecWidth == 512)
2136 IID = Intrinsic::x86_avx512_packsswb_512;
2137 else
2139 } else if (Name.starts_with("packssdw.")) {
2140 if (VecWidth == 128)
2141 IID = Intrinsic::x86_sse2_packssdw_128;
2142 else if (VecWidth == 256)
2143 IID = Intrinsic::x86_avx2_packssdw;
2144 else if (VecWidth == 512)
2145 IID = Intrinsic::x86_avx512_packssdw_512;
2146 else
2148 } else if (Name.starts_with("packuswb.")) {
2149 if (VecWidth == 128)
2150 IID = Intrinsic::x86_sse2_packuswb_128;
2151 else if (VecWidth == 256)
2152 IID = Intrinsic::x86_avx2_packuswb;
2153 else if (VecWidth == 512)
2154 IID = Intrinsic::x86_avx512_packuswb_512;
2155 else
2157 } else if (Name.starts_with("packusdw.")) {
2158 if (VecWidth == 128)
2159 IID = Intrinsic::x86_sse41_packusdw;
2160 else if (VecWidth == 256)
2161 IID = Intrinsic::x86_avx2_packusdw;
2162 else if (VecWidth == 512)
2163 IID = Intrinsic::x86_avx512_packusdw_512;
2164 else
2166 } else if (Name.starts_with("vpermilvar.")) {
2167 if (VecWidth == 128 && EltWidth == 32)
2168 IID = Intrinsic::x86_avx_vpermilvar_ps;
2169 else if (VecWidth == 128 && EltWidth == 64)
2170 IID = Intrinsic::x86_avx_vpermilvar_pd;
2171 else if (VecWidth == 256 && EltWidth == 32)
2172 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2173 else if (VecWidth == 256 && EltWidth == 64)
2174 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2175 else if (VecWidth == 512 && EltWidth == 32)
2176 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2177 else if (VecWidth == 512 && EltWidth == 64)
2178 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2179 else
2181 } else if (Name == "cvtpd2dq.256") {
2182 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2183 } else if (Name == "cvtpd2ps.256") {
2184 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2185 } else if (Name == "cvttpd2dq.256") {
2186 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2187 } else if (Name == "cvttps2dq.128") {
2188 IID = Intrinsic::x86_sse2_cvttps2dq;
2189 } else if (Name == "cvttps2dq.256") {
2190 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2191 } else if (Name.starts_with("permvar.")) {
2193 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2194 IID = Intrinsic::x86_avx2_permps;
2195 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2196 IID = Intrinsic::x86_avx2_permd;
2197 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2198 IID = Intrinsic::x86_avx512_permvar_df_256;
2199 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2200 IID = Intrinsic::x86_avx512_permvar_di_256;
2201 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2202 IID = Intrinsic::x86_avx512_permvar_sf_512;
2203 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2204 IID = Intrinsic::x86_avx512_permvar_si_512;
2205 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2206 IID = Intrinsic::x86_avx512_permvar_df_512;
2207 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2208 IID = Intrinsic::x86_avx512_permvar_di_512;
2209 else if (VecWidth == 128 && EltWidth == 16)
2210 IID = Intrinsic::x86_avx512_permvar_hi_128;
2211 else if (VecWidth == 256 && EltWidth == 16)
2212 IID = Intrinsic::x86_avx512_permvar_hi_256;
2213 else if (VecWidth == 512 && EltWidth == 16)
2214 IID = Intrinsic::x86_avx512_permvar_hi_512;
2215 else if (VecWidth == 128 && EltWidth == 8)
2216 IID = Intrinsic::x86_avx512_permvar_qi_128;
2217 else if (VecWidth == 256 && EltWidth == 8)
2218 IID = Intrinsic::x86_avx512_permvar_qi_256;
2219 else if (VecWidth == 512 && EltWidth == 8)
2220 IID = Intrinsic::x86_avx512_permvar_qi_512;
2221 else
2223 } else if (Name.starts_with("dbpsadbw.")) {
2224 if (VecWidth == 128)
2225 IID = Intrinsic::x86_avx512_dbpsadbw_128;
2226 else if (VecWidth == 256)
2227 IID = Intrinsic::x86_avx512_dbpsadbw_256;
2228 else if (VecWidth == 512)
2229 IID = Intrinsic::x86_avx512_dbpsadbw_512;
2230 else
2232 } else if (Name.starts_with("pmultishift.qb.")) {
2233 if (VecWidth == 128)
2234 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
2235 else if (VecWidth == 256)
2236 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
2237 else if (VecWidth == 512)
2238 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
2239 else
2241 } else if (Name.starts_with("conflict.")) {
2242 if (Name[9] == 'd' && VecWidth == 128)
2243 IID = Intrinsic::x86_avx512_conflict_d_128;
2244 else if (Name[9] == 'd' && VecWidth == 256)
2245 IID = Intrinsic::x86_avx512_conflict_d_256;
2246 else if (Name[9] == 'd' && VecWidth == 512)
2247 IID = Intrinsic::x86_avx512_conflict_d_512;
2248 else if (Name[9] == 'q' && VecWidth == 128)
2249 IID = Intrinsic::x86_avx512_conflict_q_128;
2250 else if (Name[9] == 'q' && VecWidth == 256)
2251 IID = Intrinsic::x86_avx512_conflict_q_256;
2252 else if (Name[9] == 'q' && VecWidth == 512)
2253 IID = Intrinsic::x86_avx512_conflict_q_512;
2254 else
2256 } else if (Name.starts_with("pavg.")) {
2257 if (Name[5] == 'b' && VecWidth == 128)
2258 IID = Intrinsic::x86_sse2_pavg_b;
2259 else if (Name[5] == 'b' && VecWidth == 256)
2260 IID = Intrinsic::x86_avx2_pavg_b;
2261 else if (Name[5] == 'b' && VecWidth == 512)
2262 IID = Intrinsic::x86_avx512_pavg_b_512;
2263 else if (Name[5] == 'w' && VecWidth == 128)
2264 IID = Intrinsic::x86_sse2_pavg_w;
2265 else if (Name[5] == 'w' && VecWidth == 256)
2266 IID = Intrinsic::x86_avx2_pavg_w;
2267 else if (Name[5] == 'w' && VecWidth == 512)
2268 IID = Intrinsic::x86_avx512_pavg_w_512;
2269 else
2271 } else
2272 return false;
2273
2275 Args.pop_back();
2276 Args.pop_back();
2278 unsigned NumArgs = CI.arg_size();
2281 return true;
2282}
2283
2284
2285
2287 size_t Pos;
2288 if (AsmStr->find("mov\tfp") == 0 &&
2289 AsmStr->find("objc_retainAutoreleaseReturnValue") != std:🧵:npos &&
2290 (Pos = AsmStr->find("# marker")) != std:🧵:npos) {
2291 AsmStr->replace(Pos, 1, ";");
2292 }
2293}
2294
2297 Value *Rep = nullptr;
2298
2299 if (Name == "abs.i" || Name == "abs.ll") {
2304 Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
2305 } else if (Name.starts_with("atomic.load.add.f32.p") ||
2306 Name.starts_with("atomic.load.add.f64.p")) {
2310 AtomicOrdering::SequentiallyConsistent);
2311 } else if (Name.consume_front("max.") &&
2312 (Name == "s" || Name == "i" || Name == "ll" || Name == "us" ||
2313 Name == "ui" || Name == "ull")) {
2316 Value *Cmp = Name.starts_with("u")
2318 : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");
2319 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");
2320 } else if (Name.consume_front("min.") &&
2321 (Name == "s" || Name == "i" || Name == "ll" || Name == "us" ||
2322 Name == "ui" || Name == "ull")) {
2325 Value *Cmp = Name.starts_with("u")
2327 : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");
2328 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");
2329 } else if (Name == "clz.ll") {
2330
2334 nullptr, "ctlz");
2336 } else if (Name == "popc.ll") {
2337
2338
2341 Arg, nullptr, "ctpop");
2343 } else if (Name == "h2f") {
2344 Rep = Builder.CreateIntrinsic(Intrinsic::convert_from_fp16,
2346 nullptr, "h2f");
2347 } else if (Name.consume_front("bitcast.") &&
2348 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
2349 Name == "d2ll")) {
2351 } else if (Name == "rotate.b32") {
2355 {Arg, Arg, ShiftAmt});
2356 } else if (Name == "rotate.b64") {
2361 {Arg, Arg, ZExtShiftAmt});
2362 } else if (Name == "rotate.right.b64") {
2367 {Arg, Arg, ZExtShiftAmt});
2368 } else if ((Name.consume_front("ptr.gen.to.") &&
2369 (Name.starts_with("local") || Name.starts_with("shared") ||
2370 Name.starts_with("global") || Name.starts_with("constant"))) ||
2371 (Name.consume_front("ptr.") &&
2372 (Name.consume_front("local") || Name.consume_front("shared") ||
2373 Name.consume_front("global") ||
2374 Name.consume_front("constant")) &&
2375 Name.starts_with(".to.gen"))) {
2377 } else if (Name.consume_front("ldg.global")) {
2379 Align PtrAlign = cast(CI->getArgOperand(1))->getAlignValue();
2380
2384 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
2385 return LD;
2386 } else {
2389 ->getReturnType()->getScalarType()->isBFloatTy()) {
2393 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
2397 Args.push_back(
2400 : Arg);
2401 }
2402 Rep = Builder.CreateCall(NewFn, Args);
2403 if (F->getReturnType()->isIntegerTy())
2404 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
2405 }
2406 }
2407
2408 return Rep;
2409}
2410
2414 Value *Rep = nullptr;
2415
2416 if (Name.starts_with("sse4a.movnt.")) {
2421
2424
2425
2426
2427 Value *Extract =
2429
2431 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
2432 } else if (Name.starts_with("avx.movnt.") ||
2433 Name.starts_with("avx512.storent.")) {
2438
2441
2443 Arg1, Arg0,
2445 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
2446 } else if (Name == "sse2.storel.dq") {
2449
2454 } else if (Name.starts_with("sse.storeu.") ||
2455 Name.starts_with("sse2.storeu.") ||
2456 Name.starts_with("avx.storeu.")) {
2460 } else if (Name == "avx512.mask.store.ss") {
2463 Mask, false);
2464 } else if (Name.starts_with("avx512.mask.store")) {
2465
2466 bool Aligned = Name[17] != 'u';
2469 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
2470
2471
2472 bool CmpEq = Name[9] == 'e';
2473 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
2476 } else if (Name.starts_with("avx512.broadcastm")) {
2484 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
2489 } else if (Name.starts_with("avx.sqrt.p") ||
2490 Name.starts_with("sse2.sqrt.p") ||
2491 Name.starts_with("sse.sqrt.p")) {
2493 {CI->getArgOperand(0)});
2494 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
2497 cast(CI->getArgOperand(3))->getZExtValue() != 4)) {
2498 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
2499 : Intrinsic::x86_avx512_sqrt_pd_512;
2500
2503 } else {
2505 {CI->getArgOperand(0)});
2506 }
2507 Rep =
2509 } else if (Name.starts_with("avx512.ptestm") ||
2510 Name.starts_with("avx512.ptestnm")) {
2514 Rep = Builder.CreateAnd(Op0, Op1);
2518 ? ICmpInst::ICMP_NE
2519 : ICmpInst::ICMP_EQ;
2520 Rep = Builder.CreateICmp(Pred, Rep, Zero);
2522 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
2524 ->getNumElements();
2526 Rep =
2528 } else if (Name.starts_with("avx512.kunpck")) {
2532 int Indices[64];
2533 for (unsigned i = 0; i != NumElts; ++i)
2534 Indices[i] = i;
2535
2536
2537
2540
2541
2544 } else if (Name == "avx512.kand.w") {
2549 } else if (Name == "avx512.kandn.w") {
2555 } else if (Name == "avx512.kor.w") {
2560 } else if (Name == "avx512.kxor.w") {
2565 } else if (Name == "avx512.kxnor.w") {
2571 } else if (Name == "avx512.knot.w") {
2575 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
2581 if (Name[14] == 'c')
2582 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
2583 else
2584 C = ConstantInt::getNullValue(Builder.getInt16Ty());
2587 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
2588 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
2589 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
2590 Name == "sse.div.ss" || Name == "sse2.div.sd") {
2593 ConstantInt::get(I32Ty, 0));
2595 ConstantInt::get(I32Ty, 0));
2597 if (Name.contains(".add."))
2598 EltOp = Builder.CreateFAdd(Elt0, Elt1);
2599 else if (Name.contains(".sub."))
2600 EltOp = Builder.CreateFSub(Elt0, Elt1);
2601 else if (Name.contains(".mul."))
2602 EltOp = Builder.CreateFMul(Elt0, Elt1);
2603 else
2604 EltOp = Builder.CreateFDiv(Elt0, Elt1);
2606 ConstantInt::get(I32Ty, 0));
2607 } else if (Name.starts_with("avx512.mask.pcmp")) {
2608
2609 bool CmpEq = Name[16] == 'e';
2611 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
2615 switch (VecWidth) {
2616 default:
2618 case 128:
2619 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
2620 break;
2621 case 256:
2622 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
2623 break;
2624 case 512:
2625 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
2626 break;
2627 }
2628
2632 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
2637 if (VecWidth == 128 && EltWidth == 32)
2638 IID = Intrinsic::x86_avx512_fpclass_ps_128;
2639 else if (VecWidth == 256 && EltWidth == 32)
2640 IID = Intrinsic::x86_avx512_fpclass_ps_256;
2641 else if (VecWidth == 512 && EltWidth == 32)
2642 IID = Intrinsic::x86_avx512_fpclass_ps_512;
2643 else if (VecWidth == 128 && EltWidth == 64)
2644 IID = Intrinsic::x86_avx512_fpclass_pd_128;
2645 else if (VecWidth == 256 && EltWidth == 64)
2646 IID = Intrinsic::x86_avx512_fpclass_pd_256;
2647 else if (VecWidth == 512 && EltWidth == 64)
2648 IID = Intrinsic::x86_avx512_fpclass_pd_512;
2649 else
2651
2655 } else if (Name.starts_with("avx512.cmp.p")) {
2657 Type *OpTy = Args[0]->getType();
2661 if (VecWidth == 128 && EltWidth == 32)
2662 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
2663 else if (VecWidth == 256 && EltWidth == 32)
2664 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
2665 else if (VecWidth == 512 && EltWidth == 32)
2666 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
2667 else if (VecWidth == 128 && EltWidth == 64)
2668 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
2669 else if (VecWidth == 256 && EltWidth == 64)
2670 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
2671 else if (VecWidth == 512 && EltWidth == 64)
2672 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
2673 else
2675
2677 if (VecWidth == 512)
2679 Args.push_back(Mask);
2680
2682 } else if (Name.starts_with("avx512.mask.cmp.")) {
2683
2684 unsigned Imm = cast(CI->getArgOperand(2))->getZExtValue();
2686 } else if (Name.starts_with("avx512.mask.ucmp.")) {
2687 unsigned Imm = cast(CI->getArgOperand(2))->getZExtValue();
2689 } else if (Name.starts_with("avx512.cvtb2mask.") ||
2690 Name.starts_with("avx512.cvtw2mask.") ||
2691 Name.starts_with("avx512.cvtd2mask.") ||
2692 Name.starts_with("avx512.cvtq2mask.")) {
2695 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
2697 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
2698 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
2699 Name.starts_with("avx512.mask.pabs")) {
2701 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
2702 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
2703 Name.starts_with("avx512.mask.pmaxs")) {
2705 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
2706 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
2707 Name.starts_with("avx512.mask.pmaxu")) {
2709 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
2710 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
2711 Name.starts_with("avx512.mask.pmins")) {
2713 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
2714 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
2715 Name.starts_with("avx512.mask.pminu")) {
2717 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
2718 Name == "avx512.pmulu.dq.512" ||
2719 Name.starts_with("avx512.mask.pmulu.dq.")) {
2720 Rep = upgradePMULDQ(Builder, *CI, false);
2721 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
2722 Name == "avx512.pmul.dq.512" ||
2723 Name.starts_with("avx512.mask.pmul.dq.")) {
2724 Rep = upgradePMULDQ(Builder, *CI, true);
2725 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
2726 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
2727 Rep =
2729 cast(CI->getType())->getElementType());
2731 } else if (Name == "avx512.cvtusi2sd") {
2732 Rep =
2734 cast(CI->getType())->getElementType());
2736 } else if (Name == "sse2.cvtss2sd") {
2739 Rep, cast(CI->getType())->getElementType());
2741 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
2742 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
2743 Name.starts_with("avx512.mask.cvtdq2pd.") ||
2744 Name.starts_with("avx512.mask.cvtudq2pd.") ||
2745 Name.starts_with("avx512.mask.cvtdq2ps.") ||
2746 Name.starts_with("avx512.mask.cvtudq2ps.") ||
2747 Name.starts_with("avx512.mask.cvtqq2pd.") ||
2748 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
2749 Name == "avx512.mask.cvtqq2ps.256" ||
2750 Name == "avx512.mask.cvtqq2ps.512" ||
2751 Name == "avx512.mask.cvtuqq2ps.256" ||
2752 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
2753 Name == "avx.cvt.ps2.pd.256" ||
2754 Name == "avx512.mask.cvtps2pd.128" ||
2755 Name == "avx512.mask.cvtps2pd.256") {
2756 auto *DstTy = cast(CI->getType());
2758 auto *SrcTy = cast(Rep->getType());
2759
2760 unsigned NumDstElts = DstTy->getNumElements();
2762 assert(NumDstElts == 2 && "Unexpected vector size");
2764 }
2765
2766 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
2767 bool IsUnsigned = Name.contains("cvtu");
2768 if (IsPS2PD)
2769 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
2770 else if (CI->arg_size() == 4 &&
2772 cast(CI->getArgOperand(3))->getZExtValue() != 4)) {
2773 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2774 : Intrinsic::x86_avx512_sitofp_round;
2777 } else {
2778 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
2780 }
2781
2785 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
2786 Name.starts_with("vcvtph2ps.")) {
2787 auto *DstTy = cast(CI->getType());
2789 auto *SrcTy = cast(Rep->getType());
2790 unsigned NumDstElts = DstTy->getNumElements();
2791 if (NumDstElts != SrcTy->getNumElements()) {
2792 assert(NumDstElts == 4 && "Unexpected vector size");
2794 }
2797 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
2801 } else if (Name.starts_with("avx512.mask.load")) {
2802
2803 bool Aligned = Name[16] != 'u';
2806 } else if (Name.starts_with("avx512.mask.expand.load.")) {
2807 auto *ResultTy = cast(CI->getType());
2809 ResultTy->getNumElements());
2810
2812 Intrinsic::masked_expandload, ResultTy,
2814 } else if (Name.starts_with("avx512.mask.compress.store.")) {
2816 Value *MaskVec =
2818 cast(ResultTy)->getNumElements());
2819
2821 Intrinsic::masked_compressstore, ResultTy,
2823 } else if (Name.starts_with("avx512.mask.compress.") ||
2824 Name.starts_with("avx512.mask.expand.")) {
2825 auto *ResultTy = cast(CI->getType());
2826
2828 ResultTy->getNumElements());
2829
2830 bool IsCompress = Name[12] == 'c';
2831 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
2832 : Intrinsic::x86_avx512_mask_expand;
2835 } else if (Name.starts_with("xop.vpcom")) {
2836 bool IsSigned;
2837 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
2838 Name.ends_with("uq"))
2839 IsSigned = false;
2840 else if (Name.ends_with("b") || Name.ends_with("w") ||
2841 Name.ends_with("d") || Name.ends_with("q"))
2842 IsSigned = true;
2843 else
2845
2846 unsigned Imm;
2848 Imm = cast(CI->getArgOperand(2))->getZExtValue();
2849 } else {
2851 if (Name.starts_with("lt"))
2852 Imm = 0;
2853 else if (Name.starts_with("le"))
2854 Imm = 1;
2855 else if (Name.starts_with("gt"))
2856 Imm = 2;
2857 else if (Name.starts_with("ge"))
2858 Imm = 3;
2859 else if (Name.starts_with("eq"))
2860 Imm = 4;
2861 else if (Name.starts_with("ne"))
2862 Imm = 5;
2863 else if (Name.starts_with("false"))
2864 Imm = 6;
2865 else if (Name.starts_with("true"))
2866 Imm = 7;
2867 else
2869 }
2870
2872 } else if (Name.starts_with("xop.vpcmov")) {
2877 Rep = Builder.CreateOr(Sel0, Sel1);
2878 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
2879 Name.starts_with("avx512.mask.prol")) {
2881 } else if (Name.starts_with("avx512.pror") ||
2882 Name.starts_with("avx512.mask.pror")) {
2884 } else if (Name.starts_with("avx512.vpshld.") ||
2885 Name.starts_with("avx512.mask.vpshld") ||
2886 Name.starts_with("avx512.maskz.vpshld")) {
2887 bool ZeroMask = Name[11] == 'z';
2889 } else if (Name.starts_with("avx512.vpshrd.") ||
2890 Name.starts_with("avx512.mask.vpshrd") ||
2891 Name.starts_with("avx512.maskz.vpshrd")) {
2892 bool ZeroMask = Name[11] == 'z';
2894 } else if (Name == "sse42.crc32.64.8") {
2897 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8, {},
2900 } else if (Name.starts_with("avx.vbroadcast.s") ||
2901 Name.starts_with("avx512.vbroadcast.s")) {
2902
2903 auto *VecTy = cast(CI->getType());
2904 Type *EltTy = VecTy->getElementType();
2905 unsigned EltNum = VecTy->getNumElements();
2909 for (unsigned I = 0; I < EltNum; ++I)
2911 } else if (Name.starts_with("sse41.pmovsx") ||
2912 Name.starts_with("sse41.pmovzx") ||
2913 Name.starts_with("avx2.pmovsx") ||
2914 Name.starts_with("avx2.pmovzx") ||
2915 Name.starts_with("avx512.mask.pmovsx") ||
2916 Name.starts_with("avx512.mask.pmovzx")) {
2917 auto *DstTy = cast(CI->getType());
2918 unsigned NumDstElts = DstTy->getNumElements();
2919
2920
2922 for (unsigned i = 0; i != NumDstElts; ++i)
2923 ShuffleMask[i] = i;
2924
2926
2927 bool DoSext = Name.contains("pmovsx");
2928 Rep =
2930
2934 } else if (Name == "avx512.mask.pmov.qd.256" ||
2935 Name == "avx512.mask.pmov.qd.512" ||
2936 Name == "avx512.mask.pmov.wb.256" ||
2937 Name == "avx512.mask.pmov.wb.512") {
2940 Rep =
2942 } else if (Name.starts_with("avx.vbroadcastf128") ||
2943 Name == "avx2.vbroadcasti128") {
2944
2945 Type *EltTy = cast(CI->getType())->getElementType();
2949 if (NumSrcElts == 2)
2951 else
2954 } else if (Name.starts_with("avx512.mask.shuf.i") ||
2955 Name.starts_with("avx512.mask.shuf.f")) {
2956 unsigned Imm = cast(CI->getArgOperand(2))->getZExtValue();
2960 unsigned ControlBitsMask = NumLanes - 1;
2961 unsigned NumControlBits = NumLanes / 2;
2963
2964 for (unsigned l = 0; l != NumLanes; ++l) {
2965 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
2966
2967 if (l >= NumLanes / 2)
2968 LaneMask += NumLanes;
2969 for (unsigned i = 0; i != NumElementsInLane; ++i)
2970 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
2971 }
2974 Rep =
2976 } else if (Name.starts_with("avx512.mask.broadcastf") ||
2977 Name.starts_with("avx512.mask.broadcasti")) {
2978 unsigned NumSrcElts = cast(CI->getArgOperand(0)->getType())
2979 ->getNumElements();
2980 unsigned NumDstElts =
2981 cast(CI->getType())->getNumElements();
2982
2984 for (unsigned i = 0; i != NumDstElts; ++i)
2985 ShuffleMask[i] = i % NumSrcElts;
2986
2989 Rep =
2991 } else if (Name.starts_with("avx2.pbroadcast") ||
2992 Name.starts_with("avx2.vbroadcast") ||
2993 Name.starts_with("avx512.pbroadcast") ||
2994 Name.starts_with("avx512.mask.broadcast.s")) {
2995
3002
3006 } else if (Name.starts_with("sse2.padds.") ||
3007 Name.starts_with("avx2.padds.") ||
3008 Name.starts_with("avx512.padds.") ||
3009 Name.starts_with("avx512.mask.padds.")) {
3011 } else if (Name.starts_with("sse2.psubs.") ||
3012 Name.starts_with("avx2.psubs.") ||
3013 Name.starts_with("avx512.psubs.") ||
3014 Name.starts_with("avx512.mask.psubs.")) {
3016 } else if (Name.starts_with("sse2.paddus.") ||
3017 Name.starts_with("avx2.paddus.") ||
3018 Name.starts_with("avx512.mask.paddus.")) {
3020 } else if (Name.starts_with("sse2.psubus.") ||
3021 Name.starts_with("avx2.psubus.") ||
3022 Name.starts_with("avx512.mask.psubus.")) {
3024 } else if (Name.starts_with("avx512.mask.palignr.")) {
3028 false);
3029 } else if (Name.starts_with("avx512.mask.valign.")) {
3033 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
3034
3035 unsigned Shift = cast(CI->getArgOperand(1))->getZExtValue();
3037 Shift / 8);
3038 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
3039
3040 unsigned Shift = cast(CI->getArgOperand(1))->getZExtValue();
3042 Shift / 8);
3043 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
3044 Name == "avx512.psll.dq.512") {
3045
3046 unsigned Shift = cast(CI->getArgOperand(1))->getZExtValue();
3048 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
3049 Name == "avx512.psrl.dq.512") {
3050
3051 unsigned Shift = cast(CI->getArgOperand(1))->getZExtValue();
3053 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
3054 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
3055 Name.starts_with("avx2.pblendd.")) {
3058 unsigned Imm = cast(CI->getArgOperand(2))->getZExtValue();
3059 auto *VecTy = cast(CI->getType());
3060 unsigned NumElts = VecTy->getNumElements();
3061
3063 for (unsigned i = 0; i != NumElts; ++i)
3064 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
3065
3067 } else if (Name.starts_with("avx.vinsertf128.") ||
3068 Name == "avx2.vinserti128" ||
3069 Name.starts_with("avx512.mask.insert")) {
3072 unsigned Imm = cast(CI->getArgOperand(2))->getZExtValue();
3073 unsigned DstNumElts =
3074 cast(CI->getType())->getNumElements();
3075 unsigned SrcNumElts =
3076 cast(Op1->getType())->getNumElements();
3077 unsigned Scale = DstNumElts / SrcNumElts;
3078
3079
3080 Imm = Imm % Scale;
3081
3082
3084 for (unsigned i = 0; i != SrcNumElts; ++i)
3085 Idxs[i] = i;
3086 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
3087 Idxs[i] = SrcNumElts;
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102 for (unsigned i = 0; i != DstNumElts; ++i)
3103 Idxs[i] = i;
3104
3105 for (unsigned i = 0; i != SrcNumElts; ++i)
3106 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
3108
3109
3113 } else if (Name.starts_with("avx.vextractf128.") ||
3114 Name == "avx2.vextracti128" ||
3115 Name.starts_with("avx512.mask.vextract")) {
3117 unsigned Imm = cast(CI->getArgOperand(1))->getZExtValue();
3118 unsigned DstNumElts =
3119 cast(CI->getType())->getNumElements();
3120 unsigned SrcNumElts =
3121 cast(Op0->getType())->getNumElements();
3122 unsigned Scale = SrcNumElts / DstNumElts;
3123
3124
3125 Imm = Imm % Scale;
3126
3127
3129 for (unsigned i = 0; i != DstNumElts; ++i) {
3130 Idxs[i] = i + (Imm * DstNumElts);
3131 }
3133
3134
3138 } else if (Name.starts_with("avx512.mask.perm.df.") ||
3139 Name.starts_with("avx512.mask.perm.di.")) {
3141 unsigned Imm = cast(CI->getArgOperand(1))->getZExtValue();
3142 auto *VecTy = cast(CI->getType());
3143 unsigned NumElts = VecTy->getNumElements();
3144
3146 for (unsigned i = 0; i != NumElts; ++i)
3147 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3148
3150
3154 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
3155
3156
3157
3158
3159
3160
3161
3162
3164
3165 unsigned NumElts = cast(CI->getType())->getNumElements();
3166 unsigned HalfSize = NumElts / 2;
3168
3169
3172
3173
3176
3177
3178 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
3179 for (unsigned i = 0; i < HalfSize; ++i)
3180 ShuffleMask[i] = StartIndex + i;
3181
3182
3183 StartIndex = (Imm & 0x10) ? HalfSize : 0;
3184 for (unsigned i = 0; i < HalfSize; ++i)
3185 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
3186
3188
3189 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
3190 Name.starts_with("avx512.mask.vpermil.p") ||
3191 Name.starts_with("avx512.mask.pshuf.d.")) {
3193 unsigned Imm = cast(CI->getArgOperand(1))->getZExtValue();
3194 auto *VecTy = cast(CI->getType());
3195 unsigned NumElts = VecTy->getNumElements();
3196
3197 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
3198 unsigned IdxMask = ((1 << IdxSize) - 1);
3199
3201
3202
3203
3204 for (unsigned i = 0; i != NumElts; ++i)
3205 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
3206
3208
3212 } else if (Name == "sse2.pshufl.w" ||
3213 Name.starts_with("avx512.mask.pshufl.w.")) {
3215 unsigned Imm = cast(CI->getArgOperand(1))->getZExtValue();
3216 unsigned NumElts = cast(CI->getType())->getNumElements();
3217
3219 for (unsigned l = 0; l != NumElts; l += 8) {
3220 for (unsigned i = 0; i != 4; ++i)
3221 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
3222 for (unsigned i = 4; i != 8; ++i)
3223 Idxs[i + l] = i + l;
3224 }
3225
3227
3231 } else if (Name == "sse2.pshufh.w" ||
3232 Name.starts_with("avx512.mask.pshufh.w.")) {
3234 unsigned Imm = cast(CI->getArgOperand(1))->getZExtValue();
3235 unsigned NumElts = cast(CI->getType())->getNumElements();
3236
3238 for (unsigned l = 0; l != NumElts; l += 8) {
3239 for (unsigned i = 0; i != 4; ++i)
3240 Idxs[i + l] = i + l;
3241 for (unsigned i = 0; i != 4; ++i)
3242 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
3243 }
3244
3246
3250 } else if (Name.starts_with("avx512.mask.shuf.p")) {
3253 unsigned Imm = cast(CI->getArgOperand(2))->getZExtValue();
3254 unsigned NumElts = cast(CI->getType())->getNumElements();
3255
3257 unsigned HalfLaneElts = NumLaneElts / 2;
3258
3260 for (unsigned i = 0; i != NumElts; ++i) {
3261
3262 Idxs[i] = i - (i % NumLaneElts);
3263
3264 if ((i % NumLaneElts) >= HalfLaneElts)
3265 Idxs[i] += NumElts;
3266
3267
3268 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
3269 }
3270
3272
3273 Rep =
3275 } else if (Name.starts_with("avx512.mask.movddup") ||
3276 Name.starts_with("avx512.mask.movshdup") ||
3277 Name.starts_with("avx512.mask.movsldup")) {
3279 unsigned NumElts = cast(CI->getType())->getNumElements();
3281
3282 unsigned Offset = 0;
3283 if (Name.starts_with("avx512.mask.movshdup."))
3285
3287 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
3288 for (unsigned i = 0; i != NumLaneElts; i += 2) {
3289 Idxs[i + l + 0] = i + l + Offset;
3290 Idxs[i + l + 1] = i + l + Offset;
3291 }
3292
3294
3295 Rep =
3297 } else if (Name.starts_with("avx512.mask.punpckl") ||
3298 Name.starts_with("avx512.mask.unpckl.")) {
3301 int NumElts = cast(CI->getType())->getNumElements();
3303
3305 for (int l = 0; l != NumElts; l += NumLaneElts)
3306 for (int i = 0; i != NumLaneElts; ++i)
3307 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
3308
3310
3311 Rep =
3313 } else if (Name.starts_with("avx512.mask.punpckh") ||
3314 Name.starts_with("avx512.mask.unpckh.")) {
3317 int NumElts = cast(CI->getType())->getNumElements();
3319
3321 for (int l = 0; l != NumElts; l += NumLaneElts)
3322 for (int i = 0; i != NumLaneElts; ++i)
3323 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
3324
3326
3327 Rep =
3329 } else if (Name.starts_with("avx512.mask.and.") ||
3330 Name.starts_with("avx512.mask.pand.")) {
3332 VectorType *ITy = VectorType::getInteger(FTy);
3336 Rep =
3338 } else if (Name.starts_with("avx512.mask.andn.") ||
3339 Name.starts_with("avx512.mask.pandn.")) {
3341 VectorType *ITy = VectorType::getInteger(FTy);
3346 Rep =
3348 } else if (Name.starts_with("avx512.mask.or.") ||
3349 Name.starts_with("avx512.mask.por.")) {
3351 VectorType *ITy = VectorType::getInteger(FTy);
3355 Rep =
3357 } else if (Name.starts_with("avx512.mask.xor.") ||
3358 Name.starts_with("avx512.mask.pxor.")) {
3360 VectorType *ITy = VectorType::getInteger(FTy);
3364 Rep =
3366 } else if (Name.starts_with("avx512.mask.padd.")) {
3368 Rep =
3370 } else if (Name.starts_with("avx512.mask.psub.")) {
3372 Rep =
3374 } else if (Name.starts_with("avx512.mask.pmull.")) {
3376 Rep =
3378 } else if (Name.starts_with("avx512.mask.add.p")) {
3379 if (Name.ends_with(".512")) {
3381 if (Name[17] == 's')
3382 IID = Intrinsic::x86_avx512_add_ps_512;
3383 else
3384 IID = Intrinsic::x86_avx512_add_pd_512;
3385
3387 IID, {},
3389 } else {
3391 }
3392 Rep =
3394 } else if (Name.starts_with("avx512.mask.div.p")) {
3395 if (Name.ends_with(".512")) {
3397 if (Name[17] == 's')
3398 IID = Intrinsic::x86_avx512_div_ps_512;
3399 else
3400 IID = Intrinsic::x86_avx512_div_pd_512;
3401
3403 IID, {},
3405 } else {
3407 }
3408 Rep =
3410 } else if (Name.starts_with("avx512.mask.mul.p")) {
3411 if (Name.ends_with(".512")) {
3413 if (Name[17] == 's')
3414 IID = Intrinsic::x86_avx512_mul_ps_512;
3415 else
3416 IID = Intrinsic::x86_avx512_mul_pd_512;
3417
3419 IID, {},
3421 } else {
3423 }
3424 Rep =
3426 } else if (Name.starts_with("avx512.mask.sub.p")) {
3427 if (Name.ends_with(".512")) {
3429 if (Name[17] == 's')
3430 IID = Intrinsic::x86_avx512_sub_ps_512;
3431 else
3432 IID = Intrinsic::x86_avx512_sub_pd_512;
3433
3435 IID, {},
3437 } else {
3439 }
3440 Rep =
3442 } else if ((Name.starts_with("avx512.mask.max.p") ||
3443 Name.starts_with("avx512.mask.min.p")) &&
3444 Name.drop_front(18) == ".512") {
3445 bool IsDouble = Name[17] == 'd';
3446 bool IsMin = Name[13] == 'i';
3448 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
3449 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
3451
3453 IID, {},
3455 Rep =
3457 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
3458 Rep =
3460 {CI->getArgOperand(0), Builder.getInt1(false)});
3461 Rep =
3463 } else if (Name.starts_with("avx512.mask.psll")) {
3464 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
3465 bool IsVariable = Name[16] == 'v';
3467 : Name[17] == '.' ? Name[18]
3468 : Name[18] == '.' ? Name[19]
3470
3472 if (IsVariable && Name[17] != '.') {
3473 if (Size == 'd' && Name[17] == '2')
3474 IID = Intrinsic::x86_avx2_psllv_q;
3475 else if (Size == 'd' && Name[17] == '4')
3476 IID = Intrinsic::x86_avx2_psllv_q_256;
3477 else if (Size == 's' && Name[17] == '4')
3478 IID = Intrinsic::x86_avx2_psllv_d;
3479 else if (Size == 's' && Name[17] == '8')
3480 IID = Intrinsic::x86_avx2_psllv_d_256;
3481 else if (Size == 'h' && Name[17] == '8')
3482 IID = Intrinsic::x86_avx512_psllv_w_128;
3483 else if (Size == 'h' && Name[17] == '1')
3484 IID = Intrinsic::x86_avx512_psllv_w_256;
3485 else if (Name[17] == '3' && Name[18] == '2')
3486 IID = Intrinsic::x86_avx512_psllv_w_512;
3487 else
3489 } else if (Name.ends_with(".128")) {
3490 if (Size == 'd')
3491 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3492 : Intrinsic::x86_sse2_psll_d;
3493 else if (Size == 'q')
3494 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3495 : Intrinsic::x86_sse2_psll_q;
3496 else if (Size == 'w')
3497 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3498 : Intrinsic::x86_sse2_psll_w;
3499 else
3501 } else if (Name.ends_with(".256")) {
3502 if (Size == 'd')
3503 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3504 : Intrinsic::x86_avx2_psll_d;
3505 else if (Size == 'q')
3506 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3507 : Intrinsic::x86_avx2_psll_q;
3508 else if (Size == 'w')
3509 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3510 : Intrinsic::x86_avx2_psll_w;
3511 else
3513 } else {
3514 if (Size == 'd')
3515 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
3516 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
3517 : Intrinsic::x86_avx512_psll_d_512;
3518 else if (Size == 'q')
3519 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
3520 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
3521 : Intrinsic::x86_avx512_psll_q_512;
3522 else if (Size == 'w')
3523 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3524 : Intrinsic::x86_avx512_psll_w_512;
3525 else
3527 }
3528
3530 } else if (Name.starts_with("avx512.mask.psrl")) {
3531 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
3532 bool IsVariable = Name[16] == 'v';
3534 : Name[17] == '.' ? Name[18]
3535 : Name[18] == '.' ? Name[19]
3537
3539 if (IsVariable && Name[17] != '.') {
3540 if (Size == 'd' && Name[17] == '2')
3541 IID = Intrinsic::x86_avx2_psrlv_q;
3542 else if (Size == 'd' && Name[17] == '4')
3543 IID = Intrinsic::x86_avx2_psrlv_q_256;
3544 else if (Size == 's' && Name[17] == '4')
3545 IID = Intrinsic::x86_avx2_psrlv_d;
3546 else if (Size == 's' && Name[17] == '8')
3547 IID = Intrinsic::x86_avx2_psrlv_d_256;
3548 else if (Size == 'h' && Name[17] == '8')
3549 IID = Intrinsic::x86_avx512_psrlv_w_128;
3550 else if (Size == 'h' && Name[17] == '1')
3551 IID = Intrinsic::x86_avx512_psrlv_w_256;
3552 else if (Name[17] == '3' && Name[18] == '2')
3553 IID = Intrinsic::x86_avx512_psrlv_w_512;
3554 else
3556 } else if (Name.ends_with(".128")) {
3557 if (Size == 'd')
3558 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
3559 : Intrinsic::x86_sse2_psrl_d;
3560 else if (Size == 'q')
3561 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
3562 : Intrinsic::x86_sse2_psrl_q;
3563 else if (Size == 'w')
3564 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
3565 : Intrinsic::x86_sse2_psrl_w;
3566 else
3568 } else if (Name.ends_with(".256")) {
3569 if (Size == 'd')
3570 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
3571 : Intrinsic::x86_avx2_psrl_d;
3572 else if (Size == 'q')
3573 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
3574 : Intrinsic::x86_avx2_psrl_q;
3575 else if (Size == 'w')
3576 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
3577 : Intrinsic::x86_avx2_psrl_w;
3578 else
3580 } else {
3581 if (Size == 'd')
3582 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
3583 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
3584 : Intrinsic::x86_avx512_psrl_d_512;
3585 else if (Size == 'q')
3586 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
3587 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
3588 : Intrinsic::x86_avx512_psrl_q_512;
3589 else if (Size == 'w')
3590 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
3591 : Intrinsic::x86_avx512_psrl_w_512;
3592 else
3594 }
3595
3597 } else if (Name.starts_with("avx512.mask.psra")) {
3598 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
3599 bool IsVariable = Name[16] == 'v';
3601 : Name[17] == '.' ? Name[18]
3602 : Name[18] == '.' ? Name[19]
3604
3606 if (IsVariable && Name[17] != '.') {
3607 if (Size == 's' && Name[17] == '4')
3608 IID = Intrinsic::x86_avx2_psrav_d;
3609 else if (Size == 's' && Name[17] == '8')
3610 IID = Intrinsic::x86_avx2_psrav_d_256;
3611 else if (Size == 'h' && Name[17] == '8')
3612 IID = Intrinsic::x86_avx512_psrav_w_128;
3613 else if (Size == 'h' && Name[17] == '1')
3614 IID = Intrinsic::x86_avx512_psrav_w_256;
3615 else if (Name[17] == '3' && Name[18] == '2')
3616 IID = Intrinsic::x86_avx512_psrav_w_512;
3617 else
3619 } else if (Name.ends_with(".128")) {
3620 if (Size == 'd')
3621 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3622 : Intrinsic::x86_sse2_psra_d;
3623 else if (Size == 'q')
3624 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
3625 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
3626 : Intrinsic::x86_avx512_psra_q_128;
3627 else if (Size == 'w')
3628 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3629 : Intrinsic::x86_sse2_psra_w;
3630 else
3632 } else if (Name.ends_with(".256")) {
3633 if (Size == 'd')
3634 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3635 : Intrinsic::x86_avx2_psra_d;
3636 else if (Size == 'q')
3637 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
3638 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
3639 : Intrinsic::x86_avx512_psra_q_256;
3640 else if (Size == 'w')
3641 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3642 : Intrinsic::x86_avx2_psra_w;
3643 else
3645 } else {
3646 if (Size == 'd')
3647 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
3648 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
3649 : Intrinsic::x86_avx512_psra_d_512;
3650 else if (Size == 'q')
3651 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
3652 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
3653 : Intrinsic::x86_avx512_psra_q_512;
3654 else if (Size == 'w')
3655 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3656 : Intrinsic::x86_avx512_psra_w_512;
3657 else
3659 }
3660
3662 } else if (Name.starts_with("avx512.mask.move.s")) {
3664 } else if (Name.starts_with("avx512.cvtmask2")) {
3666 } else if (Name.ends_with(".movntdqa")) {
3669
3674 Rep = LI;
3675 } else if (Name.starts_with("fma.vfmadd.") ||
3676 Name.starts_with("fma.vfmsub.") ||
3677 Name.starts_with("fma.vfnmadd.") ||
3678 Name.starts_with("fma.vfnmsub.")) {
3679 bool NegMul = Name[6] == 'n';
3680 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
3681 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
3682
3685
3686 if (IsScalar) {
3690 }
3691
3692 if (NegMul && !IsScalar)
3693 Ops[0] = Builder.CreateFNeg(Ops[0]);
3694 if (NegMul && IsScalar)
3695 Ops[1] = Builder.CreateFNeg(Ops[1]);
3696 if (NegAcc)
3697 Ops[2] = Builder.CreateFNeg(Ops[2]);
3698
3700
3701 if (IsScalar)
3703 } else if (Name.starts_with("fma4.vfmadd.s")) {
3706
3710
3712
3715 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
3716 Name.starts_with("avx512.maskz.vfmadd.s") ||
3717 Name.starts_with("avx512.mask3.vfmadd.s") ||
3718 Name.starts_with("avx512.mask3.vfmsub.s") ||
3719 Name.starts_with("avx512.mask3.vfnmsub.s")) {
3720 bool IsMask3 = Name[11] == '3';
3721 bool IsMaskZ = Name[11] == 'z';
3722
3723 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3724 bool NegMul = Name[2] == 'n';
3725 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
3726
3730
3731 if (NegMul && (IsMask3 || IsMaskZ))
3733 if (NegMul && !(IsMask3 || IsMaskZ))
3735 if (NegAcc)
3737
3741
3743 cast(CI->getArgOperand(4))->getZExtValue() != 4) {
3745
3747 if (Name.back() == 'd')
3748 IID = Intrinsic::x86_avx512_vfmadd_f64;
3749 else
3750 IID = Intrinsic::x86_avx512_vfmadd_f32;
3752 } else {
3753 Rep = Builder.CreateIntrinsic(Intrinsic::fma, A->getType(), {A, B, C});
3754 }
3755
3757 : IsMask3 ? C
3758 : A;
3759
3760
3761
3762 if (NegAcc && IsMask3)
3763 PassThru =
3765
3769 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
3770 Name.starts_with("avx512.mask.vfnmadd.p") ||
3771 Name.starts_with("avx512.mask.vfnmsub.p") ||
3772 Name.starts_with("avx512.mask3.vfmadd.p") ||
3773 Name.starts_with("avx512.mask3.vfmsub.p") ||
3774 Name.starts_with("avx512.mask3.vfnmsub.p") ||
3775 Name.starts_with("avx512.maskz.vfmadd.p")) {
3776 bool IsMask3 = Name[11] == '3';
3777 bool IsMaskZ = Name[11] == 'z';
3778
3779 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3780 bool NegMul = Name[2] == 'n';
3781 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
3782
3786
3787 if (NegMul && (IsMask3 || IsMaskZ))
3789 if (NegMul && !(IsMask3 || IsMaskZ))
3791 if (NegAcc)
3793
3796 cast(CI->getArgOperand(4))->getZExtValue() != 4)) {
3798
3799 if (Name[Name.size() - 5] == 's')
3800 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
3801 else
3802 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
3803
3805 } else {
3806 Rep = Builder.CreateIntrinsic(Intrinsic::fma, A->getType(), {A, B, C});
3807 }
3808
3812
3814 } else if (Name.starts_with("fma.vfmsubadd.p")) {
3818 if (VecWidth == 128 && EltWidth == 32)
3819 IID = Intrinsic::x86_fma_vfmaddsub_ps;
3820 else if (VecWidth == 256 && EltWidth == 32)
3821 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
3822 else if (VecWidth == 128 && EltWidth == 64)
3823 IID = Intrinsic::x86_fma_vfmaddsub_pd;
3824 else if (VecWidth == 256 && EltWidth == 64)
3825 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
3826 else
3828
3831 Ops[2] = Builder.CreateFNeg(Ops[2]);
3833 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
3834 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
3835 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
3836 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
3837 bool IsMask3 = Name[11] == '3';
3838 bool IsMaskZ = Name[11] == 'z';
3839
3840 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3841 bool IsSubAdd = Name[3] == 's';
3844
3845 if (Name[Name.size() - 5] == 's')
3846 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
3847 else
3848 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
3849
3852 if (IsSubAdd)
3853 Ops[2] = Builder.CreateFNeg(Ops[2]);
3854
3856 } else {
3857 int NumElts = cast(CI->getType())->getNumElements();
3858
3861
3865 Ops[2] = Builder.CreateFNeg(Ops[2]);
3867
3868 if (IsSubAdd)
3870
3872 for (int i = 0; i != NumElts; ++i)
3873 Idxs[i] = i + (i % 2) * NumElts;
3874
3876 }
3877
3881
3883 } else if (Name.starts_with("avx512.mask.pternlog.") ||
3884 Name.starts_with("avx512.maskz.pternlog.")) {
3885 bool ZeroMask = Name[11] == 'z';
3889 if (VecWidth == 128 && EltWidth == 32)
3890 IID = Intrinsic::x86_avx512_pternlog_d_128;
3891 else if (VecWidth == 256 && EltWidth == 32)
3892 IID = Intrinsic::x86_avx512_pternlog_d_256;
3893 else if (VecWidth == 512 && EltWidth == 32)
3894 IID = Intrinsic::x86_avx512_pternlog_d_512;
3895 else if (VecWidth == 128 && EltWidth == 64)
3896 IID = Intrinsic::x86_avx512_pternlog_q_128;
3897 else if (VecWidth == 256 && EltWidth == 64)
3898 IID = Intrinsic::x86_avx512_pternlog_q_256;
3899 else if (VecWidth == 512 && EltWidth == 64)
3900 IID = Intrinsic::x86_avx512_pternlog_q_512;
3901 else
3903
3910 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
3911 Name.starts_with("avx512.maskz.vpmadd52")) {
3912 bool ZeroMask = Name[11] == 'z';
3913 bool High = Name[20] == 'h' || Name[21] == 'h';
3916 if (VecWidth == 128 && )
3917 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
3918 else if (VecWidth == 256 && )
3919 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
3920 else if (VecWidth == 512 && )
3921 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
3922 else if (VecWidth == 128 && High)
3923 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
3924 else if (VecWidth == 256 && High)
3925 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
3926 else if (VecWidth == 512 && High)
3927 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
3928 else
3930
3937 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
3938 Name.starts_with("avx512.mask.vpermt2var.") ||
3939 Name.starts_with("avx512.maskz.vpermt2var.")) {
3940 bool ZeroMask = Name[11] == 'z';
3941 bool IndexForm = Name[17] == 'i';
3943 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
3944 Name.starts_with("avx512.maskz.vpdpbusd.") ||
3945 Name.starts_with("avx512.mask.vpdpbusds.") ||
3946 Name.starts_with("avx512.maskz.vpdpbusds.")) {
3947 bool ZeroMask = Name[11] == 'z';
3948 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
3951 if (VecWidth == 128 && !IsSaturating)
3952 IID = Intrinsic::x86_avx512_vpdpbusd_128;
3953 else if (VecWidth == 256 && !IsSaturating)
3954 IID = Intrinsic::x86_avx512_vpdpbusd_256;
3955 else if (VecWidth == 512 && !IsSaturating)
3956 IID = Intrinsic::x86_avx512_vpdpbusd_512;
3957 else if (VecWidth == 128 && IsSaturating)
3958 IID = Intrinsic::x86_avx512_vpdpbusds_128;
3959 else if (VecWidth == 256 && IsSaturating)
3960 IID = Intrinsic::x86_avx512_vpdpbusds_256;
3961 else if (VecWidth == 512 && IsSaturating)
3962 IID = Intrinsic::x86_avx512_vpdpbusds_512;
3963 else
3965
3972 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
3973 Name.starts_with("avx512.maskz.vpdpwssd.") ||
3974 Name.starts_with("avx512.mask.vpdpwssds.") ||
3975 Name.starts_with("avx512.maskz.vpdpwssds.")) {
3976 bool ZeroMask = Name[11] == 'z';
3977 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
3980 if (VecWidth == 128 && !IsSaturating)
3981 IID = Intrinsic::x86_avx512_vpdpwssd_128;
3982 else if (VecWidth == 256 && !IsSaturating)
3983 IID = Intrinsic::x86_avx512_vpdpwssd_256;
3984 else if (VecWidth == 512 && !IsSaturating)
3985 IID = Intrinsic::x86_avx512_vpdpwssd_512;
3986 else if (VecWidth == 128 && IsSaturating)
3987 IID = Intrinsic::x86_avx512_vpdpwssds_128;
3988 else if (VecWidth == 256 && IsSaturating)
3989 IID = Intrinsic::x86_avx512_vpdpwssds_256;
3990 else if (VecWidth == 512 && IsSaturating)
3991 IID = Intrinsic::x86_avx512_vpdpwssds_512;
3992 else
3994
4001 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
4002 Name == "addcarry.u32" || Name == "addcarry.u64" ||
4003 Name == "subborrow.u32" || Name == "subborrow.u64") {
4005 if (Name[0] == 'a' && Name.back() == '2')
4006 IID = Intrinsic::x86_addcarry_32;
4007 else if (Name[0] == 'a' && Name.back() == '4')
4008 IID = Intrinsic::x86_addcarry_64;
4009 else if (Name[0] == 's' && Name.back() == '2')
4010 IID = Intrinsic::x86_subborrow_32;
4011 else if (Name[0] == 's' && Name.back() == '4')
4012 IID = Intrinsic::x86_subborrow_64;
4013 else
4015
4016
4020
4021
4024
4026
4028 Rep = nullptr;
4029 } else if (Name.starts_with("avx512.mask.") &&
4031
4032 }
4033
4034 return Rep;
4035}
4036
4039 if (Name.starts_with("neon.bfcvt")) {
4040 if (Name.starts_with("neon.bfcvtn2")) {
4042 std::iota(LoMask.begin(), LoMask.end(), 0);
4044 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4049 } else if (Name.starts_with("neon.bfcvtn")) {
4051 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4052 Type *V4BF16 =
4055 dbgs() << "Trunc: " << *Trunc << "\n";
4058 } else {
4061 }
4062 } else if (Name.starts_with("sve.fcvt")) {
4065 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
4066 .Case("sve.fcvtnt.bf16f32",
4067 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
4071
4073
4074
4075
4078
4079 if (Args[1]->getType() != BadPredTy)
4081
4082 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
4083 BadPredTy, Args[1]);
4085 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
4086
4087 return Builder.CreateIntrinsic(NewID, {}, Args, nullptr,
4089 }
4090
4092}
4093
4096 if (Name == "mve.vctp64.old") {
4097
4098
4101 nullptr, CI->getName());
4103 Intrinsic::arm_mve_pred_v2i,
4104 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
4106 Intrinsic::arm_mve_pred_i2v,
4107 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
4108 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
4109 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
4110 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
4111 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
4113 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
4114 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
4115 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
4116 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
4118 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
4119 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
4120 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
4121 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
4122 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
4123 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
4124 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
4125 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
4126 std::vector<Type *> Tys;
4129 switch (ID) {
4130 case Intrinsic::arm_mve_mull_int_predicated:
4131 case Intrinsic::arm_mve_vqdmull_predicated:
4132 case Intrinsic::arm_mve_vldr_gather_base_predicated:
4134 break;
4135 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
4136 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
4137 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
4139 V2I1Ty};
4140 break;
4141 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
4144 break;
4145 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
4148 break;
4149 case Intrinsic::arm_cde_vcx1q_predicated:
4150 case Intrinsic::arm_cde_vcx1qa_predicated:
4151 case Intrinsic::arm_cde_vcx2q_predicated:
4152 case Intrinsic::arm_cde_vcx2qa_predicated:
4153 case Intrinsic::arm_cde_vcx3q_predicated:
4154 case Intrinsic::arm_cde_vcx3qa_predicated:
4156 break;
4157 default:
4159 }
4160
4161 std::vector<Value *> Ops;
4163 Type *Ty = Op->getType();
4166 Intrinsic::arm_mve_pred_v2i,
4167 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
4168 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
4169 }
4170 Ops.push_back(Op);
4171 }
4172
4173 return Builder.CreateIntrinsic(ID, Tys, Ops, nullptr,
4175 }
4176 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
4177}
4178
4179
4180
4181
4182
4183
4199
4201 if (NumOperands < 3)
4202 return nullptr;
4203
4205 PointerType *PtrTy = dyn_cast(Ptr->getType());
4206 if (!PtrTy)
4207 return nullptr;
4208
4211 return nullptr;
4212
4214 bool IsVolatile = false;
4215
4216
4217
4218 if (NumOperands > 3)
4219 OrderArg = dyn_cast(CI->getArgOperand(2));
4220
4221
4222
4223 if (NumOperands > 5) {
4225 IsVolatile = !VolatileArg || !VolatileArg->isZero();
4226 }
4227
4228 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
4231 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
4232 Order = AtomicOrdering::SequentiallyConsistent;
4233
4235
4236
4239 if (VT->getElementType()->isIntegerTy(16)) {
4243 }
4244 }
4245
4246
4247
4250 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
4251
4252 unsigned AddrSpace = PtrTy->getAddressSpace();
4255 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
4257 RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);
4258 }
4259
4262 MDNode *RangeNotPrivate =
4265 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
4266 }
4267
4268 if (IsVolatile)
4270
4272}
4273
4274
4275template
4278 return dyn_cast(MAV->getMetadata());
4279 return nullptr;
4280}
4281
4282
4283
4284
4287 if (Name == "label") {
4289 } else if (Name == "assign") {
4291 unwrapMAVOp(CI, 0), unwrapMAVOp(CI, 1),
4292 unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
4293 unwrapMAVOp(CI, 4), unwrapMAVOp(CI, 5),
4295 } else if (Name == "declare") {
4297 unwrapMAVOp(CI, 0), unwrapMAVOp(CI, 1),
4298 unwrapMAVOp(CI, 2), CI->getDebugLoc(),
4299 DbgVariableRecord::LocationType::Declare);
4300 } else if (Name == "addr") {
4301
4302 DIExpression *Expr = unwrapMAVOp(CI, 2);
4305 unwrapMAVOp(CI, 1), Expr,
4307 } else if (Name == "value") {
4308
4309 unsigned VarOp = 1;
4310 unsigned ExprOp = 2;
4313
4315 return;
4316 VarOp = 2;
4317 ExprOp = 3;
4318 }
4320 unwrapMAVOp(CI, 0), unwrapMAVOp(CI, VarOp),
4321 unwrapMAVOp(CI, ExprOp), CI->getDebugLoc());
4322 }
4323 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
4325}
4326
4327
4328
4330
4331
4332
4334 if ()
4335 return;
4336
4340
4341 if (!NewFn) {
4342 bool FallthroughToDefaultUpgrade = false;
4343
4345
4346 assert(Name.starts_with("llvm.") && "Intrinsic doesn't start with 'llvm.'");
4348
4349 bool IsX86 = Name.consume_front("x86.");
4350 bool IsNVVM = Name.consume_front("nvvm.");
4351 bool IsAArch64 = Name.consume_front("aarch64.");
4352 bool IsARM = Name.consume_front("arm.");
4353 bool IsAMDGCN = Name.consume_front("amdgcn.");
4354 bool IsDbg = Name.consume_front("dbg.");
4355 Value *Rep = nullptr;
4356
4357 if (!IsX86 && Name == "stackprotectorcheck") {
4358 Rep = nullptr;
4359 } else if (IsNVVM) {
4361 } else if (IsX86) {
4363 } else if (IsAArch64) {
4365 } else if (IsARM) {
4367 } else if (IsAMDGCN) {
4369 } else if (IsDbg) {
4370
4371
4372
4373
4375 bool NeedsUpgrade =
4377 if (!NeedsUpgrade)
4378 return;
4379 FallthroughToDefaultUpgrade = true;
4380 } else {
4382 }
4383 } else {
4385 }
4386
4387 if (!FallthroughToDefaultUpgrade) {
4388 if (Rep)
4391 return;
4392 }
4393 }
4394
4395 const auto &DefaultCase = [&]() -> void {
4397
4400 "Unknown function for CallBase upgrade and isn't just a name change");
4402 return;
4403 }
4404
4405
4406 if (auto *OldST = dyn_cast(CI->getType())) {
4408 "Return type must have changed");
4409 assert(OldST->getNumElements() ==
4410 cast(NewFn->getReturnType())->getNumElements() &&
4411 "Must have same number of elements");
4412
4417 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
4420 }
4423 return;
4424 }
4425
4426
4427
4430 return;
4431 };
4432 CallInst *NewCall = nullptr;
4434 default: {
4435 DefaultCase();
4436 return;
4437 }
4438 case Intrinsic::arm_neon_vst1:
4439 case Intrinsic::arm_neon_vst2:
4440 case Intrinsic::arm_neon_vst3:
4441 case Intrinsic::arm_neon_vst4:
4442 case Intrinsic::arm_neon_vst2lane:
4443 case Intrinsic::arm_neon_vst3lane:
4444 case Intrinsic::arm_neon_vst4lane: {
4446 NewCall = Builder.CreateCall(NewFn, Args);
4447 break;
4448 }
4449 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
4450 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
4451 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
4452 LLVMContext &Ctx = F->getParent()->getContext();
4455 cast(Args[3])->getZExtValue());
4456 NewCall = Builder.CreateCall(NewFn, Args);
4457 break;
4458 }
4459 case Intrinsic::aarch64_sve_ld3_sret:
4460 case Intrinsic::aarch64_sve_ld4_sret:
4461 case Intrinsic::aarch64_sve_ld2_sret: {
4469 auto *RetTy = cast(F->getReturnType());
4470 unsigned MinElts = RetTy->getMinNumElements() / N;
4474 for (unsigned I = 0; I < N; I++) {
4478 }
4479 NewCall = dyn_cast(Ret);
4480 break;
4481 }
4482
4483 case Intrinsic::coro_end: {
4486 NewCall = Builder.CreateCall(NewFn, Args);
4487 break;
4488 }
4489
4490 case Intrinsic::vector_extract: {
4493 if (.starts_with("aarch64.sve.tuple.get")) {
4494 DefaultCase();
4495 return;
4496 }
4497 auto *RetTy = cast(F->getReturnType());
4498 unsigned MinElts = RetTy->getMinNumElements();
4499 unsigned I = cast(CI->getArgOperand(1))->getZExtValue();
4502 break;
4503 }
4504
4505 case Intrinsic::vector_insert: {
4508 if (.starts_with("aarch64.sve.tuple")) {
4509 DefaultCase();
4510 return;
4511 }
4512 if (Name.starts_with("aarch64.sve.tuple.set")) {
4513 unsigned I = cast(CI->getArgOperand(1))->getZExtValue();
4519 break;
4520 }
4521 if (Name.starts_with("aarch64.sve.tuple.create")) {
4523 .StartsWith("aarch64.sve.tuple.create2", 2)
4524 .StartsWith("aarch64.sve.tuple.create3", 3)
4525 .StartsWith("aarch64.sve.tuple.create4", 4)
4527 assert(N > 1 && "Create is expected to be between 2-4");
4528 auto *RetTy = cast(F->getReturnType());
4530 unsigned MinElts = RetTy->getMinNumElements() / N;
4531 for (unsigned I = 0; I < N; I++) {
4535 }
4536 NewCall = dyn_cast(Ret);
4537 }
4538 break;
4539 }
4540
4541 case Intrinsic::arm_neon_bfdot:
4542 case Intrinsic::arm_neon_bfmmla:
4543 case Intrinsic::arm_neon_bfmlalb:
4544 case Intrinsic::arm_neon_bfmlalt:
4545 case Intrinsic::aarch64_neon_bfdot:
4546 case Intrinsic::aarch64_neon_bfmmla:
4547 case Intrinsic::aarch64_neon_bfmlalb:
4548 case Intrinsic::aarch64_neon_bfmlalt: {
4551 "Mismatch between function args and call args");
4552 size_t OperandWidth =
4554 assert((OperandWidth == 64 || OperandWidth == 128) &&
4555 "Unexpected operand width");
4557 auto Iter = CI->args().begin();
4558 Args.push_back(*Iter++);
4559 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
4560 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
4561 NewCall = Builder.CreateCall(NewFn, Args);
4562 break;
4563 }
4564
4565 case Intrinsic::bitreverse:
4567 break;
4568
4569 case Intrinsic::ctlz:
4570 case Intrinsic::cttz:
4572 "Mismatch between function args and call args");
4573 NewCall =
4575 break;
4576
4577 case Intrinsic::objectsize: {
4578 Value *NullIsUnknownSize =
4584 break;
4585 }
4586
4587 case Intrinsic::ctpop:
4589 break;
4590
4591 case Intrinsic::convert_from_fp16:
4593 break;
4594
4595 case Intrinsic::dbg_value: {
4598
4599 if (Name.starts_with("dbg.addr")) {
4601 cast(CI->getArgOperand(2))->getMetadata());
4603 NewCall =
4606 break;
4607 }
4608
4609
4611
4613 if (Offset->isZeroValue()) {
4615 NewFn,
4617 break;
4618 }
4620 return;
4621 }
4622
4623 case Intrinsic::ptr_annotation:
4624
4626 DefaultCase();
4627 return;
4628 }
4629
4630
4631 NewCall =
4638 return;
4639
4640 case Intrinsic::var_annotation:
4641
4643 DefaultCase();
4644 return;
4645 }
4646
4647 NewCall =
4654 return;
4655
4656 case Intrinsic::riscv_aes32dsi:
4657 case Intrinsic::riscv_aes32dsmi:
4658 case Intrinsic::riscv_aes32esi:
4659 case Intrinsic::riscv_aes32esmi:
4660 case Intrinsic::riscv_sm4ks:
4661 case Intrinsic::riscv_sm4ed: {
4662
4663
4666 return;
4667
4673 }
4674
4676 cast(Arg2)->getZExtValue());
4677
4678 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
4679 Value *Res = NewCall;
4685 return;
4686 }
4687 case Intrinsic::riscv_sha256sig0:
4688 case Intrinsic::riscv_sha256sig1:
4689 case Intrinsic::riscv_sha256sum0:
4690 case Intrinsic::riscv_sha256sum1:
4691 case Intrinsic::riscv_sm3p0:
4692 case Intrinsic::riscv_sm3p1: {
4693
4694
4696 return;
4697
4700
4701 NewCall = Builder.CreateCall(NewFn, Arg);
4707 return;
4708 }
4709
4710 case Intrinsic::x86_xop_vfrcz_ss:
4711 case Intrinsic::x86_xop_vfrcz_sd:
4713 break;
4714
4715 case Intrinsic::x86_xop_vpermil2pd:
4716 case Intrinsic::x86_xop_vpermil2ps:
4717 case Intrinsic::x86_xop_vpermil2pd_256:
4718 case Intrinsic::x86_xop_vpermil2ps_256: {
4721 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
4722 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
4723 NewCall = Builder.CreateCall(NewFn, Args);
4724 break;
4725 }
4726
4727 case Intrinsic::x86_sse41_ptestc:
4728 case Intrinsic::x86_sse41_ptestz:
4729 case Intrinsic::x86_sse41_ptestnzc: {
4730
4731
4732
4733
4736 return;
4737
4738
4740
4742
4745
4746 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
4747 break;
4748 }
4749
4750 case Intrinsic::x86_rdtscp: {
4751
4752
4754 return;
4755
4756 NewCall = Builder.CreateCall(NewFn);
4757
4760
4762
4766 return;
4767 }
4768
4769 case Intrinsic::x86_sse41_insertps:
4770 case Intrinsic::x86_sse41_dppd:
4771 case Intrinsic::x86_sse41_dpps:
4772 case Intrinsic::x86_sse41_mpsadbw:
4773 case Intrinsic::x86_avx_dp_ps_256:
4774 case Intrinsic::x86_avx2_mpsadbw: {
4775
4776
4778
4779
4781 NewCall = Builder.CreateCall(NewFn, Args);
4782 break;
4783 }
4784
4785 case Intrinsic::x86_avx512_mask_cmp_pd_128:
4786 case Intrinsic::x86_avx512_mask_cmp_pd_256:
4787 case Intrinsic::x86_avx512_mask_cmp_pd_512:
4788 case Intrinsic::x86_avx512_mask_cmp_ps_128:
4789 case Intrinsic::x86_avx512_mask_cmp_ps_256:
4790 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
4792 unsigned NumElts =
4793 cast(Args[0]->getType())->getNumElements();
4794 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
4795
4796 NewCall = Builder.CreateCall(NewFn, Args);
4798
4802 return;
4803 }
4804
4805 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
4806 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
4807 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
4808 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
4809 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
4810 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
4812 unsigned NumElts = cast(CI->getType())->getNumElements();
4814 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
4817
4818 NewCall = Builder.CreateCall(NewFn, Args);
4821
4825 return;
4826 }
4827 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
4828 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
4829 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
4831 unsigned NumElts =
4832 cast(CI->getType())->getNumElements() * 2;
4837
4838 NewCall = Builder.CreateCall(NewFn, Args);
4839 break;
4840 }
4841
4842 case Intrinsic::thread_pointer: {
4843 NewCall = Builder.CreateCall(NewFn, {});
4844 break;
4845 }
4846
4847 case Intrinsic::memcpy:
4848 case Intrinsic::memmove:
4849 case Intrinsic::memset: {
4850
4851
4852
4853
4854
4855
4856
4858 DefaultCase();
4859 return;
4860 }
4861
4862
4865 NewCall = Builder.CreateCall(NewFn, Args);
4869 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
4870 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
4872 auto *MemCI = cast(NewCall);
4873
4875 MemCI->setDestAlignment(Align->getMaybeAlignValue());
4876
4877 if (auto *MTI = dyn_cast(MemCI))
4878 MTI->setSourceAlignment(Align->getMaybeAlignValue());
4879 break;
4880 }
4881 }
4882 assert(NewCall && "Should have either set this variable or returned through "
4883 "the default case");
4887}
4888
4890 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
4891
4892
4893
4896
4897
4899 if (CallBase *CB = dyn_cast(U))
4901
4902
4903 F->eraseFromParent();
4904 }
4905}
4906
4909 if (NumOperands == 0)
4910 return &MD;
4911
4912
4913 if (isa(MD.getOperand(0)) && NumOperands >= 3)
4914 return &MD;
4915
4917 if (NumOperands == 3) {
4920
4921 Metadata *Elts2[] = {ScalarType, ScalarType,
4926 }
4927
4931}
4932
4935 if (Opc != Instruction::BitCast)
4936 return nullptr;
4937
4938 Temp = nullptr;
4939 Type *SrcTy = V->getType();
4943
4944
4945
4948
4949 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
4950 }
4951
4952 return nullptr;
4953}
4954
4956 if (Opc != Instruction::BitCast)
4957 return nullptr;
4958
4959 Type *SrcTy = C->getType();
4963
4964
4965
4967
4969 DestTy);
4970 }
4971
4972 return nullptr;
4973}
4974
4975
4976
4979 return false;
4980
4981
4982
4983
4984 unsigned Version = 0;
4985 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
4986 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
4987 if (Flag->getNumOperands() < 3)
4988 return false;
4989 if (MDString *K = dyn_cast_or_null(Flag->getOperand(1)))
4990 return K->getString() == "Debug Info Version";
4991 return false;
4992 });
4993 if (OpIt != ModFlags->op_end()) {
4994 const MDOperand &ValOp = (*OpIt)->getOperand(2);
4995 if (auto *CI = mdconst::dyn_extract_or_null(ValOp))
4996 Version = CI->getZExtValue();
4997 }
4998 }
4999
5001 bool BrokenDebugInfo = false;
5004 if (!BrokenDebugInfo)
5005
5006 return false;
5007 else {
5008
5010 M.getContext().diagnose(Diag);
5011 }
5012 }
5015
5017 M.getContext().diagnose(DiagVersion);
5018 }
5020}
5021
5022
5023
5025 bool Changed = false;
5026 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
5027 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
5028 if (ModRetainReleaseMarker) {
5030 if (Op) {
5031 MDString *ID = dyn_cast_or_null(Op->getOperand(0));
5032 if (ID) {
5034 ID->getString().split(ValueComp, "#");
5035 if (ValueComp.size() == 2) {
5036 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
5038 }
5040 M.eraseNamedMetadata(ModRetainReleaseMarker);
5041 Changed = true;
5042 }
5043 }
5044 }
5045 return Changed;
5046}
5047
5049
5050
5051 auto UpgradeToIntrinsic = [&](const char *OldFunc,
5053 Function *Fn = M.getFunction(OldFunc);
5054
5055 if (!Fn)
5056 return;
5057
5060
5062 CallInst *CI = dyn_cast(U);
5064 continue;
5065
5069
5070
5071
5075 continue;
5076
5077 bool InvalidCast = false;
5078
5079 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
5081
5082
5083
5084 if (I < NewFuncTy->getNumParams()) {
5085
5086
5089 InvalidCast = true;
5090 break;
5091 }
5093 }
5094 Args.push_back(Arg);
5095 }
5096
5097 if (InvalidCast)
5098 continue;
5099
5100
5102 NewCall->setTailCallKind(cast(CI)->getTailCallKind());
5104
5105
5107
5111 }
5112
5115 };
5116
5117
5118
5119 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
5120
5121
5122
5123
5125 return;
5126
5127 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
5128 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
5129 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
5130 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
5131 {"objc_autoreleaseReturnValue",
5132 llvm::Intrinsic::objc_autoreleaseReturnValue},
5133 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
5134 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
5135 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
5136 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
5137 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
5138 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
5139 {"objc_release", llvm::Intrinsic::objc_release},
5140 {"objc_retain", llvm::Intrinsic::objc_retain},
5141 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
5142 {"objc_retainAutoreleaseReturnValue",
5143 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
5144 {"objc_retainAutoreleasedReturnValue",
5145 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
5146 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
5147 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
5148 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
5149 {"objc_unsafeClaimAutoreleasedReturnValue",
5150 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
5151 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
5152 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
5153 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
5154 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
5155 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
5156 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
5157 {"objc_arc_annotation_topdown_bbstart",
5158 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
5159 {"objc_arc_annotation_topdown_bbend",
5160 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
5161 {"objc_arc_annotation_bottomup_bbstart",
5162 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
5163 {"objc_arc_annotation_bottomup_bbend",
5164 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
5165
5166 for (auto &I : RuntimeFuncs)
5167 UpgradeToIntrinsic(I.first, I.second);
5168}
5169
5171 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
5172 if (!ModFlags)
5173 return false;
5174
5175 bool HasObjCFlag = false, HasClassProperties = false, Changed = false;
5176 bool HasSwiftVersionFlag = false;
5177 uint8_t SwiftMajorVersion, SwiftMinorVersion;
5181
5182 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
5185 continue;
5186 MDString *ID = dyn_cast_or_null(Op->getOperand(1));
5187 if ()
5188 continue;
5193 Op->getOperand(2)};
5195 Changed = true;
5196 };
5197
5198 if (ID->getString() == "Objective-C Image Info Version")
5199 HasObjCFlag = true;
5200 if (ID->getString() == "Objective-C Class Properties")
5201 HasClassProperties = true;
5202
5203 if (ID->getString() == "PIC Level") {
5204 if (auto *Behavior =
5205 mdconst::dyn_extract_or_null(Op->getOperand(0))) {
5206 uint64_t V = Behavior->getLimitedValue();
5209 }
5210 }
5211
5212 if (ID->getString() == "PIE Level")
5213 if (auto *Behavior =
5214 mdconst::dyn_extract_or_null(Op->getOperand(0)))
5215 if (Behavior->getLimitedValue() == Module::Error)
5217
5218
5219
5220 if (ID->getString() == "branch-target-enforcement" ||
5221 ID->getString().starts_with("sign-return-address")) {
5222 if (auto *Behavior =
5223 mdconst::dyn_extract_or_null(Op->getOperand(0))) {
5224 if (Behavior->getLimitedValue() == Module::Error) {
5228 Op->getOperand(1), Op->getOperand(2)};
5230 Changed = true;
5231 }
5232 }
5233 }
5234
5235
5236
5237
5238 if (ID->getString() == "Objective-C Image Info Section") {
5239 if (auto *Value = dyn_cast_or_null(Op->getOperand(2))) {
5241 Value->getString().split(ValueComp, " ");
5242 if (ValueComp.size() != 1) {
5243 std::string NewValue;
5244 for (auto &S : ValueComp)
5245 NewValue += S.str();
5246 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
5249 Changed = true;
5250 }
5251 }
5252 }
5253
5254
5255
5256 if (ID->getString() == "Objective-C Garbage Collection") {
5257 auto Md = dyn_cast(Op->getOperand(2));
5258 if (Md) {
5259 assert(Md->getValue() && "Expected non-empty metadata");
5260 auto Type = Md->getValue()->getType();
5261 if (Type == Int8Ty)
5262 continue;
5263 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
5264 if ((Val & 0xff) != Val) {
5265 HasSwiftVersionFlag = true;
5266 SwiftABIVersion = (Val & 0xff00) >> 8;
5267 SwiftMajorVersion = (Val & 0xff000000) >> 24;
5268 SwiftMinorVersion = (Val & 0xff0000) >> 16;
5269 }
5272 Op->getOperand(1),
5275 Changed = true;
5276 }
5277 }
5278
5279 if (ID->getString() == "amdgpu_code_object_version") {
5281 Op->getOperand(0),
5282 MDString::get(M.getContext(), "amdhsa_code_object_version"),
5283 Op->getOperand(2)};
5285 Changed = true;
5286 }
5287 }
5288
5289
5290
5291
5292
5293
5294 if (HasObjCFlag && !HasClassProperties) {
5297 Changed = true;
5298 }
5299
5300 if (HasSwiftVersionFlag) {
5301 M.addModuleFlag(Module::Error, "Swift ABI Version",
5302 SwiftABIVersion);
5303 M.addModuleFlag(Module::Error, "Swift Major Version",
5304 ConstantInt::get(Int8Ty, SwiftMajorVersion));
5305 M.addModuleFlag(Module::Error, "Swift Minor Version",
5306 ConstantInt::get(Int8Ty, SwiftMinorVersion));
5307 Changed = true;
5308 }
5309
5310 return Changed;
5311}
5312
5314 auto TrimSpaces = [](StringRef Section) -> std::string {
5316 Section.split(Components, ',');
5317
5320
5321 for (auto Component : Components)
5322 OS << ',' << Component.trim();
5323
5324 return std::string(OS.str().substr(1));
5325 };
5326
5327 for (auto &GV : M.globals()) {
5328 if (!GV.hasSection())
5329 continue;
5330
5331 StringRef Section = GV.getSection();
5332
5333 if (!Section.starts_with("__DATA, __objc_catlist"))
5334 continue;
5335
5336
5337
5338 GV.setSection(TrimSpaces(Section));
5339 }
5340}
5341
5342namespace {
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354struct StrictFPUpgradeVisitor : public InstVisitor {
5355 StrictFPUpgradeVisitor() = default;
5356
5358 if (!Call.isStrictFP())
5359 return;
5360 if (isa(&Call))
5361 return;
5362
5363
5364 Call.removeFnAttr(Attribute::StrictFP);
5365 Call.addFnAttr(Attribute::NoBuiltin);
5366 }
5367};
5368
5369
5370struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
5371 : public InstVisitor {
5372 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
5373
5376 return;
5377
5379 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
5380 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
5381 RMW.setMetadata("amdgpu.ignore.denormal.mode", Empty);
5382 }
5383};
5384}
5385
5387
5388
5389 if (.isDeclaration() &&
.hasFnAttribute(Attribute::StrictFP)) {
5390 StrictFPUpgradeVisitor SFPV;
5391 SFPV.visit(F);
5392 }
5393
5394
5396 F.getReturnType(), F.getAttributes().getRetAttrs()));
5397 for (auto &Arg : F.args())
5398 Arg.removeAttrs(
5400
5401
5402
5403 if (Attribute A = F.getFnAttribute("implicit-section-name");
5404 A.isValid() && A.isStringAttribute()) {
5405 F.setSection(A.getValueAsString());
5406 F.removeFnAttr("implicit-section-name");
5407 }
5408
5409 if (.empty()) {
5410
5411
5412
5413 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
5414 A.isValid()) {
5415
5416 if (A.getValueAsBool()) {
5417 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
5418 Visitor.visit(F);
5419 }
5420
5421
5422
5423 F.removeFnAttr("amdgpu-unsafe-fp-atomics");
5424 }
5425 }
5426}
5427
5429 auto *T = dyn_cast_or_null(MD);
5430 if ()
5431 return false;
5432 if (T->getNumOperands() < 1)
5433 return false;
5434 auto *S = dyn_cast_or_null(T->getOperand(0));
5435 if (!S)
5436 return false;
5437 return S->getString().starts_with("llvm.vectorizer.");
5438}
5439
5441 StringRef OldPrefix = "llvm.vectorizer.";
5443
5444 if (OldTag == "llvm.vectorizer.unroll")
5445 return MDString::get(C, "llvm.loop.interleave.count");
5446
5449 .str());
5450}
5451
5453 auto *T = dyn_cast_or_null(MD);
5454 if ()
5455 return MD;
5456 if (T->getNumOperands() < 1)
5457 return MD;
5458 auto *OldTag = dyn_cast_or_null(T->getOperand(0));
5459 if (!OldTag)
5460 return MD;
5461 if (!OldTag->getString().starts_with("llvm.vectorizer."))
5462 return MD;
5463
5464
5466 Ops.reserve(T->getNumOperands());
5468 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
5470
5472}
5473
5476 if ()
5477 return &N;
5478
5480 return &N;
5481
5483 Ops.reserve(T->getNumOperands());
5484 for (Metadata *MD : T->operands())
5486
5488}
5489
5492
5493
5494 if (((T.isAMDGPU() && .isAMDGCN()) ||
5495 (T.isSPIR() || (T.isSPIRV() && .isSPIRVLogical()))) &&
5496 .contains("-G") &&
.starts_with("G")) {
5497 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
5498 }
5499
5500 if (T.isLoongArch64() || T.isRISCV64()) {
5501
5502 auto I = DL.find("-n64-");
5504 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
5505 return DL.str();
5506 }
5507
5508 std::string Res = DL.str();
5509
5510 if (T.isAMDGCN()) {
5511
5512 if (.contains("-G") &&
.starts_with("G"))
5513 Res.append(Res.empty() ? "G1" : "-G1");
5514
5515
5516
5517
5518 if (.contains("-ni") &&
.starts_with("ni"))
5519 Res.append("-ni:7:8:9");
5520
5521 if (DL.ends_with("ni:7"))
5522 Res.append(":8:9");
5523 if (DL.ends_with("ni:7:8"))
5524 Res.append(":9");
5525
5526
5527
5528 if (.contains("-p7") &&
.starts_with("p7"))
5529 Res.append("-p7:160:256:256:32");
5530 if (.contains("-p8") &&
.starts_with("p8"))
5531 Res.append("-p8:128:128");
5532 if (.contains("-p9") &&
.starts_with("p9"))
5533 Res.append("-p9:192:256:256:32");
5534
5535 return Res;
5536 }
5537
5538 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
5539
5540
5541 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
5542 if (.contains(AddrSpaces)) {
5544 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
5545 if (R.match(Res, &Groups))
5546 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
5547 }
5548 };
5549
5550
5551 if (T.isAArch64()) {
5552
5553 if (.empty() &&
.contains("-Fn32"))
5554 Res.append("-Fn32");
5555 AddPtr32Ptr64AddrSpaces();
5556 return Res;
5557 }
5558
5559 if (T.isSPARC() || (T.isMIPS64() && .contains("m:m")) || T.isPPC64() ||
5560 T.isWasm()) {
5561
5562
5563 std::string I64 = "-i64:64";
5564 std::string I128 = "-i128:128";
5566 size_t Pos = Res.find(I64);
5567 if (Pos != size_t(-1))
5568 Res.insert(Pos + I64.size(), I128);
5569 }
5570 return Res;
5571 }
5572
5573 if (.isX86())
5574 return Res;
5575
5576 AddPtr32Ptr64AddrSpaces();
5577
5578
5579
5580
5581
5582
5583
5584 if (.isOSIAMCU()) {
5585 std::string I128 = "-i128:128";
5588 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
5589 if (R.match(Res, &Groups))
5591 }
5592 }
5593
5594
5595
5596
5597 if (T.isWindowsMSVCEnvironment() && .isArch64Bit()) {
5599 auto I = Ref.find("-f80:32-");
5601 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
5602 }
5603
5604 return Res;
5605}
5606
5609 Attribute A = B.getAttribute("no-frame-pointer-elim");
5610 if (A.isValid()) {
5611
5612 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
5613 B.removeAttribute("no-frame-pointer-elim");
5614 }
5615 if (B.contains("no-frame-pointer-elim-non-leaf")) {
5616
5617 if (FramePointer != "all")
5618 FramePointer = "non-leaf";
5619 B.removeAttribute("no-frame-pointer-elim-non-leaf");
5620 }
5621 if (!FramePointer.empty())
5622 B.addAttribute("frame-pointer", FramePointer);
5623
5624 A = B.getAttribute("null-pointer-is-valid");
5625 if (A.isValid()) {
5626
5627 bool NullPointerIsValid = A.getValueAsString() == "true";
5628 B.removeAttribute("null-pointer-is-valid");
5629 if (NullPointerIsValid)
5630 B.addAttribute(Attribute::NullPointerIsValid);
5631 }
5632}
5633
5635
5636
5637
5638
5640 return OBD.getTag() == "clang.arc.attachedcall" &&
5641 OBD.inputs().empty();
5642 });
5643}
AMDGPU address space definition.
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
static Metadata * upgradeLoopArgument(Metadata *MD)
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static bool isOldLoopArgument(Metadata *MD)
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static MDType * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
static void rename(GlobalValue *GV)
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file contains constants used for implementing Dwarf debug support.
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static unsigned getNumElements(Type *Ty)
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
static const X86InstrFMA3Group Groups[]
Class for arbitrary precision integers.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Class to represent array types.
Type * getElementType() const
an instruction that atomically reads a memory location, combines it with another value,...
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ UIncWrap
Increment one up to a maximum value.
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the attributes for this call.
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
static bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
static ConstantAggregateZero * get(Type *Ty)
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
LinkageTypes getLinkage() const
Type * getValueType() const
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Value * CreateFSub(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateFDiv(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
CallInst * CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec, Value *Idx, const Twine &Name="")
Create a call to the vector.insert intrinsic.
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateFPTrunc(Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
Value * CreateICmpSGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
IntegerType * getInt16Ty()
Fetch the type representing a 16-bit integer.
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getFalse()
Get the constant value for i1 false.
Type * getFloatTy()
Fetch the type representing a 32-bit floating point value.
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Value * CreateFPExt(Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateFNeg(Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Type * getBFloatTy()
Fetch the type representing a 16-bit brain floating point value.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Base class for instruction visitors.
RetTy visitCallBase(CallBase &I)
RetTy visitAtomicRMWInst(AtomicRMWInst &I)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
This is an important class for using LLVM in a threaded context.
SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
An instruction for reading from memory.
MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
LLVMContext & getContext() const
Tracking metadata reference owned by Metadata.
static MDString * get(LLVMContext &Context, StringRef Str)
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
@ Min
Takes the min of the two values, which are required to be integers.
@ Max
Takes the max of the two values, which are required to be integers.
bool IsNewDbgInfoFormat
Is this Module using intrinsics to record the position of debugging information, or non-intrinsic rec...
void setOperand(unsigned I, MDNode *New)
MDNode * getOperand(unsigned i) const
unsigned getNumOperands() const
A container for an operand bundle being viewed as a set of values rather than a set of uses.
ArrayRef< InputTy > inputs() const
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
static ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
ArrayRef< int > getShuffleMask() const
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
size - Get the string size.
static constexpr size_t npos
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & StartsWith(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getHalfTy(LLVMContext &C)
static Type * getBFloatTy(LLVMContext &C)
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static Type * getFloatTy(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
self_iterator getIterator()
A raw_ostream that writes to an SmallVector or SmallString.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ PRIVATE_ADDRESS
Address space for private memory.
AttributeMask typeIncompatible(Type *Ty, AttributeSet AS, AttributeSafetyKind ASK=ASK_ALL)
Which attributes cannot be applied to a type.
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl< IITDescriptor > &T)
Return the IIT table descriptor for the specified intrinsic into an array of IITDescriptors.
std::optional< Function * > remangleIntrinsicFunction(Function *F)
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
AttributeList getAttributes(LLVMContext &C, ID id)
Return the attributes for an intrinsic.
This is an optimization pass for GlobalISel generic memory operations.
void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
void UpgradeSectionAttributes(Module &M)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
bool isValidAtomicOrdering(Int I)
bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Ref
The access may reference the value stored in memory.
Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
DWARFExpression::Operation Op
@ Dynamic
Denotes mode unknown at compile time.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Description of the encoding of one expression Op.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.