clang: lib/CIR/CodeGen/CIRGenBuiltinX86.cpp Source File (original) (raw)
28template <typename... Operands>
30 mlir::Location loc, const StringRef str,
31 const mlir::Type &resTy,
32 Operands &&...op) {
33 return cir::LLVMIntrinsicCallOp::create(builder, loc,
34 builder.getStringAttr(str), resTy,
35 std::forward(op)...)
36 .getResult();
37}
63 mlir::Location loc, cir::CmpOpKind pred,
64 bool shouldInvert) {
66
68 mlir::Value cmp = builder.createVecCompare(loc, pred, ops[0], ops[1]);
70 shouldInvert ? builder.createNot(cmp) : cmp, ops[0].getType());
71 return bitCast;
72}
75 mlir::Value mask, unsigned numElems) {
76 auto maskTy = cir::VectorType::get(
78 mlir::Value maskVec = builder.createBitcast(mask, maskTy);
79
80
81
82 if (numElems < 8) {
85 for (auto i : llvm::seq(0, numElems))
86 indices.push_back(cir::IntAttr::get(i32Ty, i));
87
88 maskVec = builder.createVecShuffle(loc, maskVec, maskVec, indices);
89 }
90 return maskVec;
91}
101 const mlir::Value vec,
102 const mlir::Value immediate,
103 const mlir::Location loc,
104 const bool isLow) {
106
108 unsigned numElts = vecTy.getSize();
109
110 unsigned firstHalfStart = isLow ? 0 : 4;
111 unsigned secondHalfStart = 4 - firstHalfStart;
112
113
114 imm = (imm & 0xff) * 0x01010101;
115
116 int64_t indices[32];
117 for (unsigned l = 0; l != numElts; l += 8) {
118 for (unsigned i = firstHalfStart; i != firstHalfStart + 4; ++i) {
119 indices[l + i] = l + (imm & 3) + firstHalfStart;
120 imm >>= 2;
121 }
122 for (unsigned i = secondHalfStart; i != secondHalfStart + 4; ++i)
123 indices[l + i] = l + i;
124 }
125
127}
133 uint32_t imm, const bool isShufP,
136 unsigned numElts = vecTy.getSize();
138 unsigned numLaneElts = numElts / numLanes;
139
140
141 imm = (imm & 0xff) * 0x01010101;
142
143 for (unsigned l = 0; l != numElts; l += numLaneElts) {
144 for (unsigned i = 0; i != numLaneElts; ++i) {
145 uint32_t idx = imm % numLaneElts;
146 imm /= numLaneElts;
147 if (isShufP && i >= (numLaneElts / 2))
148 idx += numElts;
149 outIndices[l + i] = l + idx;
150 }
151 }
152
153 outIndices.resize(numElts);
154}
156 mlir::Location loc, mlir::Value source,
157 mlir::Value mask,
158 mlir::Value inputVector,
159 const std::string &id) {
164 mlir::ValueRange{source, mask, maskValue});
165}
168 mlir::Location loc, mlir::Value mask,
169 unsigned numElems) {
170
171 cir::BoolType boolTy = builder.getBoolTy();
172 auto maskTy = cir::VectorType::get(
174 mlir::Value maskVec = builder.createBitcast(mask, maskTy);
175
176 if (numElems < 8) {
178 indices.reserve(numElems);
179 mlir::Type i32Ty = builder.getSInt32Ty();
180 for (auto i : llvm::seq(0, numElems))
181 indices.push_back(cir::IntAttr::get(i32Ty, i));
182
183 maskVec = builder.createVecShuffle(loc, maskVec, maskVec, indices);
184 }
185 return maskVec;
186}
189 mlir::Value mask, mlir::Value op0,
190 mlir::Value op1) {
191 auto constOp = mlir::dyn_cast_or_nullcir::ConstantOp(mask.getDefiningOp());
192
193 if (constOp && constOp.isAllOnesValue())
194 return op0;
195
198
199 return builder.createSelect(loc, mask, op0, op1);
200}
203 mlir::Location loc,
204 const std::string &intrinsicName,
206
208 unsigned numElts = intTy.getWidth();
209 mlir::Value lhsVec = getMaskVecValue(builder, loc, ops[0], numElts);
210 mlir::Value rhsVec = getMaskVecValue(builder, loc, ops[1], numElts);
211 mlir::Type vecTy = lhsVec.getType();
212 mlir::Value resVec = emitIntrinsicCallOp(builder, loc, intrinsicName, vecTy,
213 mlir::ValueRange{lhsVec, rhsVec});
215}
218 mlir::Location loc,
219 const std::string &intrinsicName,
222
223
224 mlir::Value lhs = getMaskVecValue(builder, loc, ops[0], numElems);
225 mlir::Value rhs = getMaskVecValue(builder, loc, ops[1], numElems);
226
227 mlir::Type i32Ty = builder.getSInt32Ty();
228
229
231 for (auto i : llvm::seq(0, numElems / 2))
232 halfIndices.push_back(cir::IntAttr::get(i32Ty, i));
233
234
235
236 mlir::Value lhsHalf = builder.createVecShuffle(loc, lhs, lhs, halfIndices);
237 mlir::Value rhsHalf = builder.createVecShuffle(loc, rhs, rhs, halfIndices);
238
239
240
241
242
243
244
246 for (auto i : llvm::seq(0, numElems))
247 concatIndices.push_back(cir::IntAttr::get(i32Ty, i));
248
249
250 mlir::Value res =
251 builder.createVecShuffle(loc, rhsHalf, lhsHalf, concatIndices);
253}
256 mlir::Location loc,
257 cir::BinOpKind binOpKind,
259 bool invertLHS = false) {
261 mlir::Value lhs = getMaskVecValue(builder, loc, ops[0], numElts);
262 mlir::Value rhs = getMaskVecValue(builder, loc, ops[1], numElts);
263
264 if (invertLHS)
267 ops[0].getType());
268}
271 const std::string &intrinsicName,
274 unsigned numElts = intTy.getWidth();
275 mlir::Value lhsVec = getMaskVecValue(builder, loc, ops[0], numElts);
276 mlir::Value rhsVec = getMaskVecValue(builder, loc, ops[1], numElts);
277 mlir::Type resTy = builder.getSInt32Ty();
279 mlir::ValueRange{lhsVec, rhsVec});
280}
283 mlir::Value vec, mlir::Value value,
284 mlir::Value indexOp) {
286
288 indexOp.getDefiningOpcir::ConstantOp().getIntValue().getZExtValue();
289
290 index &= numElts - 1;
291
292 cir::ConstantOp indexVal = builder.getUInt64(index, loc);
293
294 return cir::VecInsertOp::create(builder, loc, vec, value, indexVal);
295}
298 mlir::Location location, mlir::Value &op0,
299 mlir::Value &op1, mlir::Value &amt,
300 bool isRight) {
301 mlir::Type op0Ty = op0.getType();
302
303
304
305
306 if (amt.getType() != op0Ty) {
307 auto vecTy = mlir::castcir::VectorType(op0Ty);
308 uint64_t numElems = vecTy.getSize();
309
310 auto amtTy = mlir::castcir::IntType(amt.getType());
311 auto vecElemTy = mlir::castcir::IntType(vecTy.getElementType());
312
313
314
315 if (amtTy.isSigned()) {
316 cir::IntType unsignedAmtTy = builder.getUIntNTy(amtTy.getWidth());
318 }
319 cir::IntType unsignedVecElemType = builder.getUIntNTy(vecElemTy.getWidth());
320 amt = builder.createIntCast(amt, unsignedVecElemType);
321 amt = cir::VecSplatOp::create(
322 builder, location, cir::VectorType::get(unsignedVecElemType, numElems),
323 amt);
324 }
325
326 const StringRef intrinsicName = isRight ? "fshr" : "fshl";
328 mlir::ValueRange{op0, op1, amt});
329}
332 bool isSigned,
334 unsigned opTypePrimitiveSizeInBits) {
335 mlir::Type ty = cir::VectorType::get(builder.getSInt64Ty(),
336 opTypePrimitiveSizeInBits / 64);
337 mlir::Value lhs = builder.createBitcast(loc, ops[0], ty);
338 mlir::Value rhs = builder.createBitcast(loc, ops[1], ty);
339 if (isSigned) {
340 cir::ConstantOp shiftAmt =
342 cir::VecSplatOp shiftSplatVecOp =
343 cir::VecSplatOp::create(builder, loc, ty, shiftAmt.getResult());
344 mlir::Value shiftSplatValue = shiftSplatVecOp.getResult();
345
346
347
348
349 lhs = builder.createShift(loc, lhs, shiftSplatValue, true);
350 lhs = builder.createShift(loc, lhs, shiftSplatValue, false);
351 rhs = builder.createShift(loc, rhs, shiftSplatValue, true);
352 rhs = builder.createShift(loc, rhs, shiftSplatValue, false);
353 } else {
354 cir::ConstantOp maskScalar = builder.getConstant(
355 loc, cir::IntAttr::get(builder.getSInt64Ty(), 0xffffffff));
356 cir::VecSplatOp mask =
357 cir::VecSplatOp::create(builder, loc, ty, maskScalar.getResult());
358
359 lhs = builder.createAnd(loc, lhs, mask);
360 rhs = builder.createAnd(loc, rhs, mask);
361 }
362 return builder.createMul(loc, lhs, rhs);
363}
367 bool isSigned) {
368 mlir::Value op0 = ops[0];
369 mlir::Value op1 = ops[1];
370
373
375
376 cir::CmpOpKind pred;
377 switch (imm) {
378 case 0x0:
379 pred = cir::CmpOpKind::lt;
380 break;
381 case 0x1:
382 pred = cir::CmpOpKind::le;
383 break;
384 case 0x2:
385 pred = cir::CmpOpKind::gt;
386 break;
387 case 0x3:
388 pred = cir::CmpOpKind::ge;
389 break;
390 case 0x4:
391 pred = cir::CmpOpKind::eq;
392 break;
393 case 0x5:
394 pred = cir::CmpOpKind::ne;
395 break;
396 case 0x6:
397 return builder.getNullValue(ty, loc);
398 case 0x7: {
399 llvm::APInt allOnes = llvm::APInt::getAllOnes(elementTy.getWidth());
400 return cir::VecSplatOp::create(
401 builder, loc, ty,
402 builder.getConstAPInt(loc, elementTy, allOnes));
403 }
404 default:
405 llvm_unreachable("Unexpected XOP vpcom/vpcomu predicate");
406 }
407
408 if ((!isSigned && elementTy.isSigned()) ||
409 (isSigned && elementTy.isUnsigned())) {
410 elementTy = elementTy.isSigned() ? builder.getUIntNTy(elementTy.getWidth())
411 : builder.getSIntNTy(elementTy.getWidth());
412 ty = cir::VectorType::get(elementTy, ty.getSize());
415 }
416
418}
422 if (builtinID == Builtin::BI__builtin_cpu_is) {
423 cgm.errorNYI(expr->getSourceRange(), "__builtin_cpu_is");
424 return mlir::Value{};
425 }
426 if (builtinID == Builtin::BI__builtin_cpu_supports) {
427 cgm.errorNYI(expr->getSourceRange(), "__builtin_cpu_supports");
428 return mlir::Value{};
429 }
430 if (builtinID == Builtin::BI__builtin_cpu_init) {
431 cgm.errorNYI(expr->getSourceRange(), "__builtin_cpu_init");
432 return mlir::Value{};
433 }
434
435
436
438
439
441
442
444
445
446
447 unsigned iceArguments = 0;
450 assert(error == ASTContext::GE_None && "Error while getting builtin type.");
451
452 for (auto [idx, arg] : llvm::enumerate(expr->arguments()))
454
456 mlir::Type voidTy = builder.getVoidTy();
457
458 switch (builtinID) {
459 default:
460 return std::nullopt;
461 case X86::BI_mm_clflush:
463 "x86.sse2.clflush", voidTy, ops[0]);
464 case X86::BI_mm_lfence:
466 "x86.sse2.lfence", voidTy);
467 case X86::BI_mm_pause:
469 "x86.sse2.pause", voidTy);
470 case X86::BI_mm_mfence:
472 "x86.sse2.mfence", voidTy);
473 case X86::BI_mm_sfence:
475 "x86.sse.sfence", voidTy);
476 case X86::BI_mm_prefetch:
477 case X86::BI__rdtsc:
478 case X86::BI__builtin_ia32_rdtscp: {
479 cgm.errorNYI(expr->getSourceRange(),
480 std::string("unimplemented X86 builtin call: ") +
481 getContext().BuiltinInfo.getName(builtinID));
482 return mlir::Value{};
483 }
484 case X86::BI__builtin_ia32_lzcnt_u16:
485 case X86::BI__builtin_ia32_lzcnt_u32:
486 case X86::BI__builtin_ia32_lzcnt_u64: {
487 mlir::Location loc = getLoc(expr->getExprLoc());
488 mlir::Value isZeroPoison = builder.getFalse(loc);
490 mlir::ValueRange{ops[0], isZeroPoison});
491 }
492 case X86::BI__builtin_ia32_tzcnt_u16:
493 case X86::BI__builtin_ia32_tzcnt_u32:
494 case X86::BI__builtin_ia32_tzcnt_u64: {
495 mlir::Location loc = getLoc(expr->getExprLoc());
496 mlir::Value isZeroPoison = builder.getFalse(loc);
498 mlir::ValueRange{ops[0], isZeroPoison});
499 }
500 case X86::BI__builtin_ia32_undef128:
501 case X86::BI__builtin_ia32_undef256:
502 case X86::BI__builtin_ia32_undef512:
503
504
505
506
507
508 return builder.getNullValue(convertType(expr->getType()),
510 case X86::BI__builtin_ia32_vec_ext_v4hi:
511 case X86::BI__builtin_ia32_vec_ext_v16qi:
512 case X86::BI__builtin_ia32_vec_ext_v8hi:
513 case X86::BI__builtin_ia32_vec_ext_v4si:
514 case X86::BI__builtin_ia32_vec_ext_v4sf:
515 case X86::BI__builtin_ia32_vec_ext_v2di:
516 case X86::BI__builtin_ia32_vec_ext_v32qi:
517 case X86::BI__builtin_ia32_vec_ext_v16hi:
518 case X86::BI__builtin_ia32_vec_ext_v8si:
519 case X86::BI__builtin_ia32_vec_ext_v4di: {
521
523 index &= numElts - 1;
524
525 cir::ConstantOp indexVal =
527
528
529
530 return cir::VecExtractOp::create(builder, getLoc(expr->getExprLoc()),
531 ops[0], indexVal);
532 }
533 case X86::BI__builtin_ia32_vec_set_v4hi:
534 case X86::BI__builtin_ia32_vec_set_v16qi:
535 case X86::BI__builtin_ia32_vec_set_v8hi:
536 case X86::BI__builtin_ia32_vec_set_v4si:
537 case X86::BI__builtin_ia32_vec_set_v2di:
538 case X86::BI__builtin_ia32_vec_set_v32qi:
539 case X86::BI__builtin_ia32_vec_set_v16hi:
540 case X86::BI__builtin_ia32_vec_set_v8si:
541 case X86::BI__builtin_ia32_vec_set_v4di: {
543 ops[2]);
544 }
545 case X86::BI__builtin_ia32_kunpckhi:
547 "x86.avx512.kunpackb", ops);
548 case X86::BI__builtin_ia32_kunpcksi:
550 "x86.avx512.kunpackw", ops);
551 case X86::BI__builtin_ia32_kunpckdi:
553 "x86.avx512.kunpackd", ops);
554 case X86::BI_mm_setcsr:
555 case X86::BI__builtin_ia32_ldmxcsr: {
556 mlir::Location loc = getLoc(expr->getExprLoc());
558 builder.createStore(loc, ops[0], tmp);
560 builder.getVoidTy(), tmp.getPointer());
561 }
562 case X86::BI_mm_getcsr:
563 case X86::BI__builtin_ia32_stmxcsr: {
564 mlir::Location loc = getLoc(expr->getExprLoc());
566 emitIntrinsicCallOp(builder, loc, "x86.sse.stmxcsr", builder.getVoidTy(),
568 return builder.createLoad(loc, tmp);
569 }
570 case X86::BI__builtin_ia32_xsave:
571 case X86::BI__builtin_ia32_xsave64:
572 case X86::BI__builtin_ia32_xrstor:
573 case X86::BI__builtin_ia32_xrstor64:
574 case X86::BI__builtin_ia32_xsaveopt:
575 case X86::BI__builtin_ia32_xsaveopt64:
576 case X86::BI__builtin_ia32_xrstors:
577 case X86::BI__builtin_ia32_xrstors64:
578 case X86::BI__builtin_ia32_xsavec:
579 case X86::BI__builtin_ia32_xsavec64:
580 case X86::BI__builtin_ia32_xsaves:
581 case X86::BI__builtin_ia32_xsaves64:
582 case X86::BI__builtin_ia32_xsetbv:
583 case X86::BI_xsetbv: {
584 mlir::Location loc = getLoc(expr->getExprLoc());
585 StringRef intrinsicName;
586 switch (builtinID) {
587 default:
588 llvm_unreachable("Unexpected builtin");
589 case X86::BI__builtin_ia32_xsave:
590 intrinsicName = "x86.xsave";
591 break;
592 case X86::BI__builtin_ia32_xsave64:
593 intrinsicName = "x86.xsave64";
594 break;
595 case X86::BI__builtin_ia32_xrstor:
596 intrinsicName = "x86.xrstor";
597 break;
598 case X86::BI__builtin_ia32_xrstor64:
599 intrinsicName = "x86.xrstor64";
600 break;
601 case X86::BI__builtin_ia32_xsaveopt:
602 intrinsicName = "x86.xsaveopt";
603 break;
604 case X86::BI__builtin_ia32_xsaveopt64:
605 intrinsicName = "x86.xsaveopt64";
606 break;
607 case X86::BI__builtin_ia32_xrstors:
608 intrinsicName = "x86.xrstors";
609 break;
610 case X86::BI__builtin_ia32_xrstors64:
611 intrinsicName = "x86.xrstors64";
612 break;
613 case X86::BI__builtin_ia32_xsavec:
614 intrinsicName = "x86.xsavec";
615 break;
616 case X86::BI__builtin_ia32_xsavec64:
617 intrinsicName = "x86.xsavec64";
618 break;
619 case X86::BI__builtin_ia32_xsaves:
620 intrinsicName = "x86.xsaves";
621 break;
622 case X86::BI__builtin_ia32_xsaves64:
623 intrinsicName = "x86.xsaves64";
624 break;
625 case X86::BI__builtin_ia32_xsetbv:
626 case X86::BI_xsetbv:
627 intrinsicName = "x86.xsetbv";
628 break;
629 }
630
631
632
633
634
635 mlir::Type i32Ty = builder.getSInt32Ty();
636
637
638 cir::ConstantOp shift32 = builder.getSInt64(32, loc);
639 mlir::Value mhi = builder.createShift(loc, ops[1], shift32.getResult(),
640 false);
641 mhi = builder.createIntCast(mhi, i32Ty);
642
643
644 mlir::Value mlo = builder.createIntCast(ops[1], i32Ty);
645
647 mlir::ValueRange{ops[0], mhi, mlo});
648 }
649 case X86::BI__builtin_ia32_xgetbv:
650 case X86::BI_xgetbv:
651
652
654 "x86.xgetbv", builder.getUInt64Ty(), ops[0]);
655 case X86::BI__builtin_ia32_storedqudi128_mask:
656 case X86::BI__builtin_ia32_storedqusi128_mask:
657 case X86::BI__builtin_ia32_storedquhi128_mask:
658 case X86::BI__builtin_ia32_storedquqi128_mask:
659 case X86::BI__builtin_ia32_storeupd128_mask:
660 case X86::BI__builtin_ia32_storeups128_mask:
661 case X86::BI__builtin_ia32_storedqudi256_mask:
662 case X86::BI__builtin_ia32_storedqusi256_mask:
663 case X86::BI__builtin_ia32_storedquhi256_mask:
664 case X86::BI__builtin_ia32_storedquqi256_mask:
665 case X86::BI__builtin_ia32_storeupd256_mask:
666 case X86::BI__builtin_ia32_storeups256_mask:
667 case X86::BI__builtin_ia32_storedqudi512_mask:
668 case X86::BI__builtin_ia32_storedqusi512_mask:
669 case X86::BI__builtin_ia32_storedquhi512_mask:
670 case X86::BI__builtin_ia32_storedquqi512_mask:
671 case X86::BI__builtin_ia32_storeupd512_mask:
672 case X86::BI__builtin_ia32_storeups512_mask:
673 case X86::BI__builtin_ia32_storesbf16128_mask:
674 case X86::BI__builtin_ia32_storesh128_mask:
675 case X86::BI__builtin_ia32_storess128_mask:
676 case X86::BI__builtin_ia32_storesd128_mask:
677 case X86::BI__builtin_ia32_cvtmask2b128:
678 case X86::BI__builtin_ia32_cvtmask2b256:
679 case X86::BI__builtin_ia32_cvtmask2b512:
680 case X86::BI__builtin_ia32_cvtmask2w128:
681 case X86::BI__builtin_ia32_cvtmask2w256:
682 case X86::BI__builtin_ia32_cvtmask2w512:
683 case X86::BI__builtin_ia32_cvtmask2d128:
684 case X86::BI__builtin_ia32_cvtmask2d256:
685 case X86::BI__builtin_ia32_cvtmask2d512:
686 case X86::BI__builtin_ia32_cvtmask2q128:
687 case X86::BI__builtin_ia32_cvtmask2q256:
688 case X86::BI__builtin_ia32_cvtmask2q512:
689 case X86::BI__builtin_ia32_cvtb2mask128:
690 case X86::BI__builtin_ia32_cvtb2mask256:
691 case X86::BI__builtin_ia32_cvtb2mask512:
692 case X86::BI__builtin_ia32_cvtw2mask128:
693 case X86::BI__builtin_ia32_cvtw2mask256:
694 case X86::BI__builtin_ia32_cvtw2mask512:
695 case X86::BI__builtin_ia32_cvtd2mask128:
696 case X86::BI__builtin_ia32_cvtd2mask256:
697 case X86::BI__builtin_ia32_cvtd2mask512:
698 case X86::BI__builtin_ia32_cvtq2mask128:
699 case X86::BI__builtin_ia32_cvtq2mask256:
700 case X86::BI__builtin_ia32_cvtq2mask512:
701 case X86::BI__builtin_ia32_cvtdq2ps512_mask:
702 case X86::BI__builtin_ia32_cvtqq2ps512_mask:
703 case X86::BI__builtin_ia32_cvtqq2pd512_mask:
704 case X86::BI__builtin_ia32_vcvtw2ph512_mask:
705 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
706 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
707 case X86::BI__builtin_ia32_cvtudq2ps512_mask:
708 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
709 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
710 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
711 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
712 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
713 case X86::BI__builtin_ia32_vfmaddsh3_mask:
714 case X86::BI__builtin_ia32_vfmaddss3_mask:
715 case X86::BI__builtin_ia32_vfmaddsd3_mask:
716 case X86::BI__builtin_ia32_vfmaddsh3_maskz:
717 case X86::BI__builtin_ia32_vfmaddss3_maskz:
718 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
719 case X86::BI__builtin_ia32_vfmaddsh3_mask3:
720 case X86::BI__builtin_ia32_vfmaddss3_mask3:
721 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
722 case X86::BI__builtin_ia32_vfmsubsh3_mask3:
723 case X86::BI__builtin_ia32_vfmsubss3_mask3:
724 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
725 case X86::BI__builtin_ia32_vfmaddph512_mask:
726 case X86::BI__builtin_ia32_vfmaddph512_maskz:
727 case X86::BI__builtin_ia32_vfmaddph512_mask3:
728 case X86::BI__builtin_ia32_vfmaddps512_mask:
729 case X86::BI__builtin_ia32_vfmaddps512_maskz:
730 case X86::BI__builtin_ia32_vfmaddps512_mask3:
731 case X86::BI__builtin_ia32_vfmsubps512_mask3:
732 case X86::BI__builtin_ia32_vfmaddpd512_mask:
733 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
734 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
735 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
736 case X86::BI__builtin_ia32_vfmsubph512_mask3:
737 case X86::BI__builtin_ia32_vfmaddsubph512_mask:
738 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
739 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
740 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
741 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
742 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
743 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
744 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
745 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
746 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
747 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
748 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
749 case X86::BI__builtin_ia32_movdqa32store128_mask:
750 case X86::BI__builtin_ia32_movdqa64store128_mask:
751 case X86::BI__builtin_ia32_storeaps128_mask:
752 case X86::BI__builtin_ia32_storeapd128_mask:
753 case X86::BI__builtin_ia32_movdqa32store256_mask:
754 case X86::BI__builtin_ia32_movdqa64store256_mask:
755 case X86::BI__builtin_ia32_storeaps256_mask:
756 case X86::BI__builtin_ia32_storeapd256_mask:
757 case X86::BI__builtin_ia32_movdqa32store512_mask:
758 case X86::BI__builtin_ia32_movdqa64store512_mask:
759 case X86::BI__builtin_ia32_storeaps512_mask:
760 case X86::BI__builtin_ia32_storeapd512_mask:
761 case X86::BI__builtin_ia32_loadups128_mask:
762 case X86::BI__builtin_ia32_loadups256_mask:
763 case X86::BI__builtin_ia32_loadups512_mask:
764 case X86::BI__builtin_ia32_loadupd128_mask:
765 case X86::BI__builtin_ia32_loadupd256_mask:
766 case X86::BI__builtin_ia32_loadupd512_mask:
767 case X86::BI__builtin_ia32_loaddquqi128_mask:
768 case X86::BI__builtin_ia32_loaddquqi256_mask:
769 case X86::BI__builtin_ia32_loaddquqi512_mask:
770 case X86::BI__builtin_ia32_loaddquhi128_mask:
771 case X86::BI__builtin_ia32_loaddquhi256_mask:
772 case X86::BI__builtin_ia32_loaddquhi512_mask:
773 case X86::BI__builtin_ia32_loaddqusi128_mask:
774 case X86::BI__builtin_ia32_loaddqusi256_mask:
775 case X86::BI__builtin_ia32_loaddqusi512_mask:
776 case X86::BI__builtin_ia32_loaddqudi128_mask:
777 case X86::BI__builtin_ia32_loaddqudi256_mask:
778 case X86::BI__builtin_ia32_loaddqudi512_mask:
779 case X86::BI__builtin_ia32_loadsbf16128_mask:
780 case X86::BI__builtin_ia32_loadsh128_mask:
781 case X86::BI__builtin_ia32_loadss128_mask:
782 case X86::BI__builtin_ia32_loadsd128_mask:
783 case X86::BI__builtin_ia32_loadaps128_mask:
784 case X86::BI__builtin_ia32_loadaps256_mask:
785 case X86::BI__builtin_ia32_loadaps512_mask:
786 case X86::BI__builtin_ia32_loadapd128_mask:
787 case X86::BI__builtin_ia32_loadapd256_mask:
788 case X86::BI__builtin_ia32_loadapd512_mask:
789 case X86::BI__builtin_ia32_movdqa32load128_mask:
790 case X86::BI__builtin_ia32_movdqa32load256_mask:
791 case X86::BI__builtin_ia32_movdqa32load512_mask:
792 case X86::BI__builtin_ia32_movdqa64load128_mask:
793 case X86::BI__builtin_ia32_movdqa64load256_mask:
794 case X86::BI__builtin_ia32_movdqa64load512_mask:
795 case X86::BI__builtin_ia32_expandloaddf128_mask:
796 case X86::BI__builtin_ia32_expandloaddf256_mask:
797 case X86::BI__builtin_ia32_expandloaddf512_mask:
798 case X86::BI__builtin_ia32_expandloadsf128_mask:
799 case X86::BI__builtin_ia32_expandloadsf256_mask:
800 case X86::BI__builtin_ia32_expandloadsf512_mask:
801 case X86::BI__builtin_ia32_expandloaddi128_mask:
802 case X86::BI__builtin_ia32_expandloaddi256_mask:
803 case X86::BI__builtin_ia32_expandloaddi512_mask:
804 case X86::BI__builtin_ia32_expandloadsi128_mask:
805 case X86::BI__builtin_ia32_expandloadsi256_mask:
806 case X86::BI__builtin_ia32_expandloadsi512_mask:
807 case X86::BI__builtin_ia32_expandloadhi128_mask:
808 case X86::BI__builtin_ia32_expandloadhi256_mask:
809 case X86::BI__builtin_ia32_expandloadhi512_mask:
810 case X86::BI__builtin_ia32_expandloadqi128_mask:
811 case X86::BI__builtin_ia32_expandloadqi256_mask:
812 case X86::BI__builtin_ia32_expandloadqi512_mask:
813 case X86::BI__builtin_ia32_compressstoredf128_mask:
814 case X86::BI__builtin_ia32_compressstoredf256_mask:
815 case X86::BI__builtin_ia32_compressstoredf512_mask:
816 case X86::BI__builtin_ia32_compressstoresf128_mask:
817 case X86::BI__builtin_ia32_compressstoresf256_mask:
818 case X86::BI__builtin_ia32_compressstoresf512_mask:
819 case X86::BI__builtin_ia32_compressstoredi128_mask:
820 case X86::BI__builtin_ia32_compressstoredi256_mask:
821 case X86::BI__builtin_ia32_compressstoredi512_mask:
822 case X86::BI__builtin_ia32_compressstoresi128_mask:
823 case X86::BI__builtin_ia32_compressstoresi256_mask:
824 case X86::BI__builtin_ia32_compressstoresi512_mask:
825 case X86::BI__builtin_ia32_compressstorehi128_mask:
826 case X86::BI__builtin_ia32_compressstorehi256_mask:
827 case X86::BI__builtin_ia32_compressstorehi512_mask:
828 case X86::BI__builtin_ia32_compressstoreqi128_mask:
829 case X86::BI__builtin_ia32_compressstoreqi256_mask:
830 case X86::BI__builtin_ia32_compressstoreqi512_mask:
831 cgm.errorNYI(expr->getSourceRange(),
832 std::string("unimplemented X86 builtin call: ") +
833 getContext().BuiltinInfo.getName(builtinID));
834 return mlir::Value{};
835 case X86::BI__builtin_ia32_expanddf128_mask:
836 case X86::BI__builtin_ia32_expanddf256_mask:
837 case X86::BI__builtin_ia32_expanddf512_mask:
838 case X86::BI__builtin_ia32_expandsf128_mask:
839 case X86::BI__builtin_ia32_expandsf256_mask:
840 case X86::BI__builtin_ia32_expandsf512_mask:
841 case X86::BI__builtin_ia32_expanddi128_mask:
842 case X86::BI__builtin_ia32_expanddi256_mask:
843 case X86::BI__builtin_ia32_expanddi512_mask:
844 case X86::BI__builtin_ia32_expandsi128_mask:
845 case X86::BI__builtin_ia32_expandsi256_mask:
846 case X86::BI__builtin_ia32_expandsi512_mask:
847 case X86::BI__builtin_ia32_expandhi128_mask:
848 case X86::BI__builtin_ia32_expandhi256_mask:
849 case X86::BI__builtin_ia32_expandhi512_mask:
850 case X86::BI__builtin_ia32_expandqi128_mask:
851 case X86::BI__builtin_ia32_expandqi256_mask:
852 case X86::BI__builtin_ia32_expandqi512_mask: {
853 mlir::Location loc = getLoc(expr->getExprLoc());
855 "x86.avx512.mask.expand");
856 }
857 case X86::BI__builtin_ia32_compressdf128_mask:
858 case X86::BI__builtin_ia32_compressdf256_mask:
859 case X86::BI__builtin_ia32_compressdf512_mask:
860 case X86::BI__builtin_ia32_compresssf128_mask:
861 case X86::BI__builtin_ia32_compresssf256_mask:
862 case X86::BI__builtin_ia32_compresssf512_mask:
863 case X86::BI__builtin_ia32_compressdi128_mask:
864 case X86::BI__builtin_ia32_compressdi256_mask:
865 case X86::BI__builtin_ia32_compressdi512_mask:
866 case X86::BI__builtin_ia32_compresssi128_mask:
867 case X86::BI__builtin_ia32_compresssi256_mask:
868 case X86::BI__builtin_ia32_compresssi512_mask:
869 case X86::BI__builtin_ia32_compresshi128_mask:
870 case X86::BI__builtin_ia32_compresshi256_mask:
871 case X86::BI__builtin_ia32_compresshi512_mask:
872 case X86::BI__builtin_ia32_compressqi128_mask:
873 case X86::BI__builtin_ia32_compressqi256_mask:
874 case X86::BI__builtin_ia32_compressqi512_mask: {
875 mlir::Location loc = getLoc(expr->getExprLoc());
877 "x86.avx512.mask.compress");
878 }
879 case X86::BI__builtin_ia32_gather3div2df:
880 case X86::BI__builtin_ia32_gather3div2di:
881 case X86::BI__builtin_ia32_gather3div4df:
882 case X86::BI__builtin_ia32_gather3div4di:
883 case X86::BI__builtin_ia32_gather3div4sf:
884 case X86::BI__builtin_ia32_gather3div4si:
885 case X86::BI__builtin_ia32_gather3div8sf:
886 case X86::BI__builtin_ia32_gather3div8si:
887 case X86::BI__builtin_ia32_gather3siv2df:
888 case X86::BI__builtin_ia32_gather3siv2di:
889 case X86::BI__builtin_ia32_gather3siv4df:
890 case X86::BI__builtin_ia32_gather3siv4di:
891 case X86::BI__builtin_ia32_gather3siv4sf:
892 case X86::BI__builtin_ia32_gather3siv4si:
893 case X86::BI__builtin_ia32_gather3siv8sf:
894 case X86::BI__builtin_ia32_gather3siv8si:
895 case X86::BI__builtin_ia32_gathersiv8df:
896 case X86::BI__builtin_ia32_gathersiv16sf:
897 case X86::BI__builtin_ia32_gatherdiv8df:
898 case X86::BI__builtin_ia32_gatherdiv16sf:
899 case X86::BI__builtin_ia32_gathersiv8di:
900 case X86::BI__builtin_ia32_gathersiv16si:
901 case X86::BI__builtin_ia32_gatherdiv8di:
902 case X86::BI__builtin_ia32_gatherdiv16si: {
903 StringRef intrinsicName;
904 switch (builtinID) {
905 default:
906 llvm_unreachable("Unexpected builtin");
907 case X86::BI__builtin_ia32_gather3div2df:
908 intrinsicName = "x86.avx512.mask.gather3div2.df";
909 break;
910 case X86::BI__builtin_ia32_gather3div2di:
911 intrinsicName = "x86.avx512.mask.gather3div2.di";
912 break;
913 case X86::BI__builtin_ia32_gather3div4df:
914 intrinsicName = "x86.avx512.mask.gather3div4.df";
915 break;
916 case X86::BI__builtin_ia32_gather3div4di:
917 intrinsicName = "x86.avx512.mask.gather3div4.di";
918 break;
919 case X86::BI__builtin_ia32_gather3div4sf:
920 intrinsicName = "x86.avx512.mask.gather3div4.sf";
921 break;
922 case X86::BI__builtin_ia32_gather3div4si:
923 intrinsicName = "x86.avx512.mask.gather3div4.si";
924 break;
925 case X86::BI__builtin_ia32_gather3div8sf:
926 intrinsicName = "x86.avx512.mask.gather3div8.sf";
927 break;
928 case X86::BI__builtin_ia32_gather3div8si:
929 intrinsicName = "x86.avx512.mask.gather3div8.si";
930 break;
931 case X86::BI__builtin_ia32_gather3siv2df:
932 intrinsicName = "x86.avx512.mask.gather3siv2.df";
933 break;
934 case X86::BI__builtin_ia32_gather3siv2di:
935 intrinsicName = "x86.avx512.mask.gather3siv2.di";
936 break;
937 case X86::BI__builtin_ia32_gather3siv4df:
938 intrinsicName = "x86.avx512.mask.gather3siv4.df";
939 break;
940 case X86::BI__builtin_ia32_gather3siv4di:
941 intrinsicName = "x86.avx512.mask.gather3siv4.di";
942 break;
943 case X86::BI__builtin_ia32_gather3siv4sf:
944 intrinsicName = "x86.avx512.mask.gather3siv4.sf";
945 break;
946 case X86::BI__builtin_ia32_gather3siv4si:
947 intrinsicName = "x86.avx512.mask.gather3siv4.si";
948 break;
949 case X86::BI__builtin_ia32_gather3siv8sf:
950 intrinsicName = "x86.avx512.mask.gather3siv8.sf";
951 break;
952 case X86::BI__builtin_ia32_gather3siv8si:
953 intrinsicName = "x86.avx512.mask.gather3siv8.si";
954 break;
955 case X86::BI__builtin_ia32_gathersiv8df:
956 intrinsicName = "x86.avx512.mask.gather.dpd.512";
957 break;
958 case X86::BI__builtin_ia32_gathersiv16sf:
959 intrinsicName = "x86.avx512.mask.gather.dps.512";
960 break;
961 case X86::BI__builtin_ia32_gatherdiv8df:
962 intrinsicName = "x86.avx512.mask.gather.qpd.512";
963 break;
964 case X86::BI__builtin_ia32_gatherdiv16sf:
965 intrinsicName = "x86.avx512.mask.gather.qps.512";
966 break;
967 case X86::BI__builtin_ia32_gathersiv8di:
968 intrinsicName = "x86.avx512.mask.gather.dpq.512";
969 break;
970 case X86::BI__builtin_ia32_gathersiv16si:
971 intrinsicName = "x86.avx512.mask.gather.dpi.512";
972 break;
973 case X86::BI__builtin_ia32_gatherdiv8di:
974 intrinsicName = "x86.avx512.mask.gather.qpq.512";
975 break;
976 case X86::BI__builtin_ia32_gatherdiv16si:
977 intrinsicName = "x86.avx512.mask.gather.qpi.512";
978 break;
979 }
980
981 mlir::Location loc = getLoc(expr->getExprLoc());
982 unsigned minElts =
988 }
989 case X86::BI__builtin_ia32_scattersiv8df:
990 case X86::BI__builtin_ia32_scattersiv16sf:
991 case X86::BI__builtin_ia32_scatterdiv8df:
992 case X86::BI__builtin_ia32_scatterdiv16sf:
993 case X86::BI__builtin_ia32_scattersiv8di:
994 case X86::BI__builtin_ia32_scattersiv16si:
995 case X86::BI__builtin_ia32_scatterdiv8di:
996 case X86::BI__builtin_ia32_scatterdiv16si:
997 case X86::BI__builtin_ia32_scatterdiv2df:
998 case X86::BI__builtin_ia32_scatterdiv2di:
999 case X86::BI__builtin_ia32_scatterdiv4df:
1000 case X86::BI__builtin_ia32_scatterdiv4di:
1001 case X86::BI__builtin_ia32_scatterdiv4sf:
1002 case X86::BI__builtin_ia32_scatterdiv4si:
1003 case X86::BI__builtin_ia32_scatterdiv8sf:
1004 case X86::BI__builtin_ia32_scatterdiv8si:
1005 case X86::BI__builtin_ia32_scattersiv2df:
1006 case X86::BI__builtin_ia32_scattersiv2di:
1007 case X86::BI__builtin_ia32_scattersiv4df:
1008 case X86::BI__builtin_ia32_scattersiv4di:
1009 case X86::BI__builtin_ia32_scattersiv4sf:
1010 case X86::BI__builtin_ia32_scattersiv4si:
1011 case X86::BI__builtin_ia32_scattersiv8sf:
1012 case X86::BI__builtin_ia32_scattersiv8si: {
1013 llvm::StringRef intrinsicName;
1014 switch (builtinID) {
1015 default:
1016 llvm_unreachable("Unexpected builtin");
1017 case X86::BI__builtin_ia32_scattersiv8df:
1018 intrinsicName = "x86.avx512.mask.scatter.dpd.512";
1019 break;
1020 case X86::BI__builtin_ia32_scattersiv16sf:
1021 intrinsicName = "x86.avx512.mask.scatter.dps.512";
1022 break;
1023 case X86::BI__builtin_ia32_scatterdiv8df:
1024 intrinsicName = "x86.avx512.mask.scatter.qpd.512";
1025 break;
1026 case X86::BI__builtin_ia32_scatterdiv16sf:
1027 intrinsicName = "x86.avx512.mask.scatter.qps.512";
1028 break;
1029 case X86::BI__builtin_ia32_scattersiv8di:
1030 intrinsicName = "x86.avx512.mask.scatter.dpq.512";
1031 break;
1032 case X86::BI__builtin_ia32_scattersiv16si:
1033 intrinsicName = "x86.avx512.mask.scatter.dpi.512";
1034 break;
1035 case X86::BI__builtin_ia32_scatterdiv8di:
1036 intrinsicName = "x86.avx512.mask.scatter.qpq.512";
1037 break;
1038 case X86::BI__builtin_ia32_scatterdiv16si:
1039 intrinsicName = "x86.avx512.mask.scatter.qpi.512";
1040 break;
1041 case X86::BI__builtin_ia32_scatterdiv2df:
1042 intrinsicName = "x86.avx512.mask.scatterdiv2.df";
1043 break;
1044 case X86::BI__builtin_ia32_scatterdiv2di:
1045 intrinsicName = "x86.avx512.mask.scatterdiv2.di";
1046 break;
1047 case X86::BI__builtin_ia32_scatterdiv4df:
1048 intrinsicName = "x86.avx512.mask.scatterdiv4.df";
1049 break;
1050 case X86::BI__builtin_ia32_scatterdiv4di:
1051 intrinsicName = "x86.avx512.mask.scatterdiv4.di";
1052 break;
1053 case X86::BI__builtin_ia32_scatterdiv4sf:
1054 intrinsicName = "x86.avx512.mask.scatterdiv4.sf";
1055 break;
1056 case X86::BI__builtin_ia32_scatterdiv4si:
1057 intrinsicName = "x86.avx512.mask.scatterdiv4.si";
1058 break;
1059 case X86::BI__builtin_ia32_scatterdiv8sf:
1060 intrinsicName = "x86.avx512.mask.scatterdiv8.sf";
1061 break;
1062 case X86::BI__builtin_ia32_scatterdiv8si:
1063 intrinsicName = "x86.avx512.mask.scatterdiv8.si";
1064 break;
1065 case X86::BI__builtin_ia32_scattersiv2df:
1066 intrinsicName = "x86.avx512.mask.scattersiv2.df";
1067 break;
1068 case X86::BI__builtin_ia32_scattersiv2di:
1069 intrinsicName = "x86.avx512.mask.scattersiv2.di";
1070 break;
1071 case X86::BI__builtin_ia32_scattersiv4df:
1072 intrinsicName = "x86.avx512.mask.scattersiv4.df";
1073 break;
1074 case X86::BI__builtin_ia32_scattersiv4di:
1075 intrinsicName = "x86.avx512.mask.scattersiv4.di";
1076 break;
1077 case X86::BI__builtin_ia32_scattersiv4sf:
1078 intrinsicName = "x86.avx512.mask.scattersiv4.sf";
1079 break;
1080 case X86::BI__builtin_ia32_scattersiv4si:
1081 intrinsicName = "x86.avx512.mask.scattersiv4.si";
1082 break;
1083 case X86::BI__builtin_ia32_scattersiv8sf:
1084 intrinsicName = "x86.avx512.mask.scattersiv8.sf";
1085 break;
1086 case X86::BI__builtin_ia32_scattersiv8si:
1087 intrinsicName = "x86.avx512.mask.scattersiv8.si";
1088 break;
1089 }
1090
1091 mlir::Location loc = getLoc(expr->getExprLoc());
1092 unsigned minElts =
1095 ops[1] = getMaskVecValue(builder, loc, ops[1], minElts);
1096
1099 }
1100 case X86::BI__builtin_ia32_vextractf128_pd256:
1101 case X86::BI__builtin_ia32_vextractf128_ps256:
1102 case X86::BI__builtin_ia32_vextractf128_si256:
1103 case X86::BI__builtin_ia32_extract128i256:
1104 case X86::BI__builtin_ia32_extractf64x4_mask:
1105 case X86::BI__builtin_ia32_extractf32x4_mask:
1106 case X86::BI__builtin_ia32_extracti64x4_mask:
1107 case X86::BI__builtin_ia32_extracti32x4_mask:
1108 case X86::BI__builtin_ia32_extractf32x8_mask:
1109 case X86::BI__builtin_ia32_extracti32x8_mask:
1110 case X86::BI__builtin_ia32_extractf32x4_256_mask:
1111 case X86::BI__builtin_ia32_extracti32x4_256_mask:
1112 case X86::BI__builtin_ia32_extractf64x2_256_mask:
1113 case X86::BI__builtin_ia32_extracti64x2_256_mask:
1114 case X86::BI__builtin_ia32_extractf64x2_512_mask:
1115 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
1116 mlir::Location loc = getLoc(expr->getExprLoc());
1118 unsigned numElts = dstTy.getSize();
1120 unsigned subVectors = srcNumElts / numElts;
1121 assert(llvm::isPowerOf2_32(subVectors) && "Expected power of 2 subvectors");
1122 unsigned index =
1123 ops[1].getDefiningOpcir::ConstantOp().getIntValue().getZExtValue();
1124
1125 index &= subVectors - 1;
1126 index *= numElts;
1127
1128 int64_t indices[16];
1129 std::iota(indices, indices + numElts, index);
1130
1131 mlir::Value poison =
1132 builder.getConstant(loc, cir::PoisonAttr::get(ops[0].getType()));
1133 mlir::Value res = builder.createVecShuffle(loc, ops[0], poison,
1134 ArrayRef(indices, numElts));
1135 if (ops.size() == 4)
1136 res = emitX86Select(builder, loc, ops[3], res, ops[2]);
1137
1138 return res;
1139 }
1140 case X86::BI__builtin_ia32_vinsertf128_pd256:
1141 case X86::BI__builtin_ia32_vinsertf128_ps256:
1142 case X86::BI__builtin_ia32_vinsertf128_si256:
1143 case X86::BI__builtin_ia32_insert128i256:
1144 case X86::BI__builtin_ia32_insertf64x4:
1145 case X86::BI__builtin_ia32_insertf32x4:
1146 case X86::BI__builtin_ia32_inserti64x4:
1147 case X86::BI__builtin_ia32_inserti32x4:
1148 case X86::BI__builtin_ia32_insertf32x8:
1149 case X86::BI__builtin_ia32_inserti32x8:
1150 case X86::BI__builtin_ia32_insertf32x4_256:
1151 case X86::BI__builtin_ia32_inserti32x4_256:
1152 case X86::BI__builtin_ia32_insertf64x2_256:
1153 case X86::BI__builtin_ia32_inserti64x2_256:
1154 case X86::BI__builtin_ia32_insertf64x2_512:
1155 case X86::BI__builtin_ia32_inserti64x2_512:
1156 case X86::BI__builtin_ia32_pmovqd512_mask:
1157 case X86::BI__builtin_ia32_pmovwb512_mask:
1158 case X86::BI__builtin_ia32_pblendw128:
1159 case X86::BI__builtin_ia32_blendpd:
1160 case X86::BI__builtin_ia32_blendps:
1161 case X86::BI__builtin_ia32_blendpd256:
1162 case X86::BI__builtin_ia32_blendps256:
1163 case X86::BI__builtin_ia32_pblendw256:
1164 case X86::BI__builtin_ia32_pblendd128:
1165 case X86::BI__builtin_ia32_pblendd256:
1166 cgm.errorNYI(expr->getSourceRange(),
1167 std::string("unimplemented X86 builtin call: ") +
1168 getContext().BuiltinInfo.getName(builtinID));
1169 return mlir::Value{};
1170 case X86::BI__builtin_ia32_pshuflw:
1171 case X86::BI__builtin_ia32_pshuflw256:
1172 case X86::BI__builtin_ia32_pshuflw512:
1174 true);
1175 case X86::BI__builtin_ia32_pshufhw:
1176 case X86::BI__builtin_ia32_pshufhw256:
1177 case X86::BI__builtin_ia32_pshufhw512:
1179 false);
1180 case X86::BI__builtin_ia32_pshufd:
1181 case X86::BI__builtin_ia32_pshufd256:
1182 case X86::BI__builtin_ia32_pshufd512:
1183 case X86::BI__builtin_ia32_vpermilpd:
1184 case X86::BI__builtin_ia32_vpermilps:
1185 case X86::BI__builtin_ia32_vpermilpd256:
1186 case X86::BI__builtin_ia32_vpermilps256:
1187 case X86::BI__builtin_ia32_vpermilpd512:
1188 case X86::BI__builtin_ia32_vpermilps512: {
1190
1193
1194 return builder.createVecShuffle(getLoc(expr->getExprLoc()), ops[0], mask);
1195 }
1196 case X86::BI__builtin_ia32_shufpd:
1197 case X86::BI__builtin_ia32_shufpd256:
1198 case X86::BI__builtin_ia32_shufpd512:
1199 case X86::BI__builtin_ia32_shufps:
1200 case X86::BI__builtin_ia32_shufps256:
1201 case X86::BI__builtin_ia32_shufps512: {
1203
1206
1207 return builder.createVecShuffle(getLoc(expr->getExprLoc()), ops[0], ops[1],
1208 mask);
1209 }
1210 case X86::BI__builtin_ia32_permdi256:
1211 case X86::BI__builtin_ia32_permdf256:
1212 case X86::BI__builtin_ia32_permdi512:
1213 case X86::BI__builtin_ia32_permdf512:
1214 case X86::BI__builtin_ia32_palignr128:
1215 case X86::BI__builtin_ia32_palignr256:
1216 case X86::BI__builtin_ia32_palignr512:
1217 cgm.errorNYI(expr->getSourceRange(),
1218 std::string("unimplemented X86 builtin call: ") +
1219 getContext().BuiltinInfo.getName(builtinID));
1220 return {};
1221 case X86::BI__builtin_ia32_alignd128:
1222 case X86::BI__builtin_ia32_alignd256:
1223 case X86::BI__builtin_ia32_alignd512:
1224 case X86::BI__builtin_ia32_alignq128:
1225 case X86::BI__builtin_ia32_alignq256:
1226 case X86::BI__builtin_ia32_alignq512: {
1228 unsigned shiftVal =
1229 ops[2].getDefiningOpcir::ConstantOp().getIntValue().getZExtValue() &
1230 0xff;
1231
1232
1233 shiftVal &= numElts - 1;
1234
1236 mlir::Type i32Ty = builder.getSInt32Ty();
1237 for (unsigned i = 0; i != numElts; ++i)
1238 indices.push_back(cir::IntAttr::get(i32Ty, i + shiftVal));
1239
1240 return builder.createVecShuffle(getLoc(expr->getExprLoc()), ops[0], ops[1],
1241 indices);
1242 }
1243 case X86::BI__builtin_ia32_shuf_f32x4_256:
1244 case X86::BI__builtin_ia32_shuf_f64x2_256:
1245 case X86::BI__builtin_ia32_shuf_i32x4_256:
1246 case X86::BI__builtin_ia32_shuf_i64x2_256:
1247 case X86::BI__builtin_ia32_shuf_f32x4:
1248 case X86::BI__builtin_ia32_shuf_f64x2:
1249 case X86::BI__builtin_ia32_shuf_i32x4:
1250 case X86::BI__builtin_ia32_shuf_i64x2:
1251 case X86::BI__builtin_ia32_vperm2f128_pd256:
1252 case X86::BI__builtin_ia32_vperm2f128_ps256:
1253 case X86::BI__builtin_ia32_vperm2f128_si256:
1254 case X86::BI__builtin_ia32_permti256:
1255 case X86::BI__builtin_ia32_pslldqi128_byteshift:
1256 case X86::BI__builtin_ia32_pslldqi256_byteshift:
1257 case X86::BI__builtin_ia32_pslldqi512_byteshift:
1258 case X86::BI__builtin_ia32_psrldqi128_byteshift:
1259 case X86::BI__builtin_ia32_psrldqi256_byteshift:
1260 case X86::BI__builtin_ia32_psrldqi512_byteshift:
1261 cgm.errorNYI(expr->getSourceRange(),
1262 std::string("unimplemented X86 builtin call: ") +
1263 getContext().BuiltinInfo.getName(builtinID));
1264 return mlir::Value{};
1265 case X86::BI__builtin_ia32_kshiftliqi:
1266 case X86::BI__builtin_ia32_kshiftlihi:
1267 case X86::BI__builtin_ia32_kshiftlisi:
1268 case X86::BI__builtin_ia32_kshiftlidi: {
1269 mlir::Location loc = getLoc(expr->getExprLoc());
1270 unsigned shiftVal =
1271 ops[1].getDefiningOpcir::ConstantOp().getIntValue().getZExtValue() &
1272 0xff;
1274
1275 if (shiftVal >= numElems)
1276 return builder.getNullValue(ops[0].getType(), loc);
1277
1278 mlir::Value in = getMaskVecValue(builder, loc, ops[0], numElems);
1279
1281 mlir::Type i32Ty = builder.getSInt32Ty();
1282 for (auto i : llvm::seq(0, numElems))
1283 indices.push_back(cir::IntAttr::get(i32Ty, numElems + i - shiftVal));
1284
1285 mlir::Value zero = builder.getNullValue(in.getType(), loc);
1286 mlir::Value sv = builder.createVecShuffle(loc, zero, in, indices);
1287 return builder.createBitcast(sv, ops[0].getType());
1288 }
1289 case X86::BI__builtin_ia32_kshiftriqi:
1290 case X86::BI__builtin_ia32_kshiftrihi:
1291 case X86::BI__builtin_ia32_kshiftrisi:
1292 case X86::BI__builtin_ia32_kshiftridi: {
1293 mlir::Location loc = getLoc(expr->getExprLoc());
1294 unsigned shiftVal =
1295 ops[1].getDefiningOpcir::ConstantOp().getIntValue().getZExtValue() &
1296 0xff;
1298
1299 if (shiftVal >= numElems)
1300 return builder.getNullValue(ops[0].getType(), loc);
1301
1302 mlir::Value in = getMaskVecValue(builder, loc, ops[0], numElems);
1303
1305 mlir::Type i32Ty = builder.getSInt32Ty();
1306 for (auto i : llvm::seq(0, numElems))
1307 indices.push_back(cir::IntAttr::get(i32Ty, i + shiftVal));
1308
1309 mlir::Value zero = builder.getNullValue(in.getType(), loc);
1310 mlir::Value sv = builder.createVecShuffle(loc, in, zero, indices);
1311 return builder.createBitcast(sv, ops[0].getType());
1312 }
1313 case X86::BI__builtin_ia32_vprotbi:
1314 case X86::BI__builtin_ia32_vprotwi:
1315 case X86::BI__builtin_ia32_vprotdi:
1316 case X86::BI__builtin_ia32_vprotqi:
1317 case X86::BI__builtin_ia32_prold128:
1318 case X86::BI__builtin_ia32_prold256:
1319 case X86::BI__builtin_ia32_prold512:
1320 case X86::BI__builtin_ia32_prolq128:
1321 case X86::BI__builtin_ia32_prolq256:
1322 case X86::BI__builtin_ia32_prolq512:
1324 ops[0], ops[1], false);
1325 case X86::BI__builtin_ia32_prord128:
1326 case X86::BI__builtin_ia32_prord256:
1327 case X86::BI__builtin_ia32_prord512:
1328 case X86::BI__builtin_ia32_prorq128:
1329 case X86::BI__builtin_ia32_prorq256:
1330 case X86::BI__builtin_ia32_prorq512:
1332 ops[0], ops[1], true);
1333 case X86::BI__builtin_ia32_selectb_128:
1334 case X86::BI__builtin_ia32_selectb_256:
1335 case X86::BI__builtin_ia32_selectb_512:
1336 case X86::BI__builtin_ia32_selectw_128:
1337 case X86::BI__builtin_ia32_selectw_256:
1338 case X86::BI__builtin_ia32_selectw_512:
1339 case X86::BI__builtin_ia32_selectd_128:
1340 case X86::BI__builtin_ia32_selectd_256:
1341 case X86::BI__builtin_ia32_selectd_512:
1342 case X86::BI__builtin_ia32_selectq_128:
1343 case X86::BI__builtin_ia32_selectq_256:
1344 case X86::BI__builtin_ia32_selectq_512:
1345 case X86::BI__builtin_ia32_selectph_128:
1346 case X86::BI__builtin_ia32_selectph_256:
1347 case X86::BI__builtin_ia32_selectph_512:
1348 case X86::BI__builtin_ia32_selectpbf_128:
1349 case X86::BI__builtin_ia32_selectpbf_256:
1350 case X86::BI__builtin_ia32_selectpbf_512:
1351 case X86::BI__builtin_ia32_selectps_128:
1352 case X86::BI__builtin_ia32_selectps_256:
1353 case X86::BI__builtin_ia32_selectps_512:
1354 case X86::BI__builtin_ia32_selectpd_128:
1355 case X86::BI__builtin_ia32_selectpd_256:
1356 case X86::BI__builtin_ia32_selectpd_512:
1357 case X86::BI__builtin_ia32_selectsh_128:
1358 case X86::BI__builtin_ia32_selectsbf_128:
1359 case X86::BI__builtin_ia32_selectss_128:
1360 case X86::BI__builtin_ia32_selectsd_128:
1361 case X86::BI__builtin_ia32_cmpb128_mask:
1362 case X86::BI__builtin_ia32_cmpb256_mask:
1363 case X86::BI__builtin_ia32_cmpb512_mask:
1364 case X86::BI__builtin_ia32_cmpw128_mask:
1365 case X86::BI__builtin_ia32_cmpw256_mask:
1366 case X86::BI__builtin_ia32_cmpw512_mask:
1367 case X86::BI__builtin_ia32_cmpd128_mask:
1368 case X86::BI__builtin_ia32_cmpd256_mask:
1369 case X86::BI__builtin_ia32_cmpd512_mask:
1370 case X86::BI__builtin_ia32_cmpq128_mask:
1371 case X86::BI__builtin_ia32_cmpq256_mask:
1372 case X86::BI__builtin_ia32_cmpq512_mask:
1373 case X86::BI__builtin_ia32_ucmpb128_mask:
1374 case X86::BI__builtin_ia32_ucmpb256_mask:
1375 case X86::BI__builtin_ia32_ucmpb512_mask:
1376 case X86::BI__builtin_ia32_ucmpw128_mask:
1377 case X86::BI__builtin_ia32_ucmpw256_mask:
1378 case X86::BI__builtin_ia32_ucmpw512_mask:
1379 case X86::BI__builtin_ia32_ucmpd128_mask:
1380 case X86::BI__builtin_ia32_ucmpd256_mask:
1381 case X86::BI__builtin_ia32_ucmpd512_mask:
1382 case X86::BI__builtin_ia32_ucmpq128_mask:
1383 case X86::BI__builtin_ia32_ucmpq256_mask:
1384 case X86::BI__builtin_ia32_ucmpq512_mask:
1385 cgm.errorNYI(expr->getSourceRange(),
1386 std::string("unimplemented X86 builtin call: ") +
1387 getContext().BuiltinInfo.getName(builtinID));
1388 return mlir::Value{};
1389 case X86::BI__builtin_ia32_vpcomb:
1390 case X86::BI__builtin_ia32_vpcomw:
1391 case X86::BI__builtin_ia32_vpcomd:
1392 case X86::BI__builtin_ia32_vpcomq:
1394 case X86::BI__builtin_ia32_vpcomub:
1395 case X86::BI__builtin_ia32_vpcomuw:
1396 case X86::BI__builtin_ia32_vpcomud:
1397 case X86::BI__builtin_ia32_vpcomuq:
1399 case X86::BI__builtin_ia32_kortestcqi:
1400 case X86::BI__builtin_ia32_kortestchi:
1401 case X86::BI__builtin_ia32_kortestcsi:
1402 case X86::BI__builtin_ia32_kortestcdi: {
1403 mlir::Location loc = getLoc(expr->getExprLoc());
1405 mlir::Value allOnesOp =
1406 builder.getConstAPInt(loc, ty, APInt::getAllOnes(ty.getWidth()));
1407 mlir::Value orOp = emitX86MaskLogic(builder, loc, cir::BinOpKind::Or, ops);
1408 mlir::Value cmp =
1409 cir::CmpOp::create(builder, loc, cir::CmpOpKind::eq, orOp, allOnesOp);
1410 return builder.createCast(cir::CastKind::bool_to_int, cmp,
1411 cgm.convertType(expr->getType()));
1412 }
1413 case X86::BI__builtin_ia32_kortestzqi:
1414 case X86::BI__builtin_ia32_kortestzhi:
1415 case X86::BI__builtin_ia32_kortestzsi:
1416 case X86::BI__builtin_ia32_kortestzdi: {
1417 mlir::Location loc = getLoc(expr->getExprLoc());
1419 mlir::Value allZerosOp = builder.getNullValue(ty, loc).getResult();
1420 mlir::Value orOp = emitX86MaskLogic(builder, loc, cir::BinOpKind::Or, ops);
1421 mlir::Value cmp =
1422 cir::CmpOp::create(builder, loc, cir::CmpOpKind::eq, orOp, allZerosOp);
1423 return builder.createCast(cir::CastKind::bool_to_int, cmp,
1424 cgm.convertType(expr->getType()));
1425 }
1426 case X86::BI__builtin_ia32_ktestcqi:
1428 "x86.avx512.ktestc.b", ops);
1429 case X86::BI__builtin_ia32_ktestzqi:
1431 "x86.avx512.ktestz.b", ops);
1432 case X86::BI__builtin_ia32_ktestchi:
1434 "x86.avx512.ktestc.w", ops);
1435 case X86::BI__builtin_ia32_ktestzhi:
1437 "x86.avx512.ktestz.w", ops);
1438 case X86::BI__builtin_ia32_ktestcsi:
1440 "x86.avx512.ktestc.d", ops);
1441 case X86::BI__builtin_ia32_ktestzsi:
1443 "x86.avx512.ktestz.d", ops);
1444 case X86::BI__builtin_ia32_ktestcdi:
1446 "x86.avx512.ktestc.q", ops);
1447 case X86::BI__builtin_ia32_ktestzdi:
1449 "x86.avx512.ktestz.q", ops);
1450 case X86::BI__builtin_ia32_kaddqi:
1452 "x86.avx512.kadd.b", ops);
1453 case X86::BI__builtin_ia32_kaddhi:
1455 "x86.avx512.kadd.w", ops);
1456 case X86::BI__builtin_ia32_kaddsi:
1458 "x86.avx512.kadd.d", ops);
1459 case X86::BI__builtin_ia32_kadddi:
1461 "x86.avx512.kadd.q", ops);
1462 case X86::BI__builtin_ia32_kandqi:
1463 case X86::BI__builtin_ia32_kandhi:
1464 case X86::BI__builtin_ia32_kandsi:
1465 case X86::BI__builtin_ia32_kanddi:
1467 cir::BinOpKind::And, ops);
1468 case X86::BI__builtin_ia32_kandnqi:
1469 case X86::BI__builtin_ia32_kandnhi:
1470 case X86::BI__builtin_ia32_kandnsi:
1471 case X86::BI__builtin_ia32_kandndi:
1473 cir::BinOpKind::And, ops, true);
1474 case X86::BI__builtin_ia32_korqi:
1475 case X86::BI__builtin_ia32_korhi:
1476 case X86::BI__builtin_ia32_korsi:
1477 case X86::BI__builtin_ia32_kordi:
1479 cir::BinOpKind::Or, ops);
1480 case X86::BI__builtin_ia32_kxnorqi:
1481 case X86::BI__builtin_ia32_kxnorhi:
1482 case X86::BI__builtin_ia32_kxnorsi:
1483 case X86::BI__builtin_ia32_kxnordi:
1485 cir::BinOpKind::Xor, ops, true);
1486 case X86::BI__builtin_ia32_kxorqi:
1487 case X86::BI__builtin_ia32_kxorhi:
1488 case X86::BI__builtin_ia32_kxorsi:
1489 case X86::BI__builtin_ia32_kxordi:
1491 cir::BinOpKind::Xor, ops);
1492 case X86::BI__builtin_ia32_knotqi:
1493 case X86::BI__builtin_ia32_knothi:
1494 case X86::BI__builtin_ia32_knotsi:
1495 case X86::BI__builtin_ia32_knotdi: {
1497 unsigned numElts = intTy.getWidth();
1498 mlir::Value resVec =
1500 return builder.createBitcast(builder.createNot(resVec), ops[0].getType());
1501 }
1502 case X86::BI__builtin_ia32_kmovb:
1503 case X86::BI__builtin_ia32_kmovw:
1504 case X86::BI__builtin_ia32_kmovd:
1505 case X86::BI__builtin_ia32_kmovq: {
1506
1507
1508
1510 unsigned numElts = intTy.getWidth();
1511 mlir::Value resVec =
1513 return builder.createBitcast(resVec, ops[0].getType());
1514 }
1515 case X86::BI__builtin_ia32_sqrtsh_round_mask:
1516 case X86::BI__builtin_ia32_sqrtsd_round_mask:
1517 case X86::BI__builtin_ia32_sqrtss_round_mask:
1518 cgm.errorNYI(expr->getSourceRange(),
1519 std::string("unimplemented X86 builtin call: ") +
1520 getContext().BuiltinInfo.getName(builtinID));
1521 return mlir::Value{};
1522 case X86::BI__builtin_ia32_sqrtph512:
1523 case X86::BI__builtin_ia32_sqrtps512:
1524 case X86::BI__builtin_ia32_sqrtpd512: {
1525 mlir::Location loc = getLoc(expr->getExprLoc());
1526 mlir::Value arg = ops[0];
1527 return cir::SqrtOp::create(builder, loc, arg.getType(), arg).getResult();
1528 }
1529 case X86::BI__builtin_ia32_pmuludq128:
1530 case X86::BI__builtin_ia32_pmuludq256:
1531 case X86::BI__builtin_ia32_pmuludq512: {
1532 unsigned opTypePrimitiveSizeInBits =
1533 cgm.getDataLayout().getTypeSizeInBits(ops[0].getType());
1535 ops, opTypePrimitiveSizeInBits);
1536 }
1537 case X86::BI__builtin_ia32_pmuldq128:
1538 case X86::BI__builtin_ia32_pmuldq256:
1539 case X86::BI__builtin_ia32_pmuldq512: {
1540 unsigned opTypePrimitiveSizeInBits =
1541 cgm.getDataLayout().getTypeSizeInBits(ops[0].getType());
1543 ops, opTypePrimitiveSizeInBits);
1544 }
1545 case X86::BI__builtin_ia32_pternlogd512_mask:
1546 case X86::BI__builtin_ia32_pternlogq512_mask:
1547 case X86::BI__builtin_ia32_pternlogd128_mask:
1548 case X86::BI__builtin_ia32_pternlogd256_mask:
1549 case X86::BI__builtin_ia32_pternlogq128_mask:
1550 case X86::BI__builtin_ia32_pternlogq256_mask:
1551 case X86::BI__builtin_ia32_pternlogd512_maskz:
1552 case X86::BI__builtin_ia32_pternlogq512_maskz:
1553 case X86::BI__builtin_ia32_pternlogd128_maskz:
1554 case X86::BI__builtin_ia32_pternlogd256_maskz:
1555 case X86::BI__builtin_ia32_pternlogq128_maskz:
1556 case X86::BI__builtin_ia32_pternlogq256_maskz:
1557 case X86::BI__builtin_ia32_vpshldd128:
1558 case X86::BI__builtin_ia32_vpshldd256:
1559 case X86::BI__builtin_ia32_vpshldd512:
1560 case X86::BI__builtin_ia32_vpshldq128:
1561 case X86::BI__builtin_ia32_vpshldq256:
1562 case X86::BI__builtin_ia32_vpshldq512:
1563 case X86::BI__builtin_ia32_vpshldw128:
1564 case X86::BI__builtin_ia32_vpshldw256:
1565 case X86::BI__builtin_ia32_vpshldw512:
1566 case X86::BI__builtin_ia32_vpshrdd128:
1567 case X86::BI__builtin_ia32_vpshrdd256:
1568 case X86::BI__builtin_ia32_vpshrdd512:
1569 case X86::BI__builtin_ia32_vpshrdq128:
1570 case X86::BI__builtin_ia32_vpshrdq256:
1571 case X86::BI__builtin_ia32_vpshrdq512:
1572 case X86::BI__builtin_ia32_vpshrdw128:
1573 case X86::BI__builtin_ia32_vpshrdw256:
1574 case X86::BI__builtin_ia32_vpshrdw512:
1575 cgm.errorNYI(expr->getSourceRange(),
1576 std::string("unimplemented X86 builtin call: ") +
1577 getContext().BuiltinInfo.getName(builtinID));
1578 return {};
1579 case X86::BI__builtin_ia32_reduce_fadd_pd512:
1580 case X86::BI__builtin_ia32_reduce_fadd_ps512:
1581 case X86::BI__builtin_ia32_reduce_fadd_ph512:
1582 case X86::BI__builtin_ia32_reduce_fadd_ph256:
1583 case X86::BI__builtin_ia32_reduce_fadd_ph128: {
1586 "vector.reduce.fadd", ops[0].getType(),
1587 mlir::ValueRange{ops[0], ops[1]});
1588 }
1589 case X86::BI__builtin_ia32_reduce_fmul_pd512:
1590 case X86::BI__builtin_ia32_reduce_fmul_ps512:
1591 case X86::BI__builtin_ia32_reduce_fmul_ph512:
1592 case X86::BI__builtin_ia32_reduce_fmul_ph256:
1593 case X86::BI__builtin_ia32_reduce_fmul_ph128: {
1596 "vector.reduce.fmul", ops[0].getType(),
1597 mlir::ValueRange{ops[0], ops[1]});
1598 }
1599 case X86::BI__builtin_ia32_reduce_fmax_pd512:
1600 case X86::BI__builtin_ia32_reduce_fmax_ps512:
1601 case X86::BI__builtin_ia32_reduce_fmax_ph512:
1602 case X86::BI__builtin_ia32_reduce_fmax_ph256:
1603 case X86::BI__builtin_ia32_reduce_fmax_ph128: {
1607 "vector.reduce.fmax", vecTy.getElementType(),
1608 mlir::ValueRange{ops[0]});
1609 }
1610 case X86::BI__builtin_ia32_reduce_fmin_pd512:
1611 case X86::BI__builtin_ia32_reduce_fmin_ps512:
1612 case X86::BI__builtin_ia32_reduce_fmin_ph512:
1613 case X86::BI__builtin_ia32_reduce_fmin_ph256:
1614 case X86::BI__builtin_ia32_reduce_fmin_ph128: {
1618 "vector.reduce.fmin", vecTy.getElementType(),
1619 mlir::ValueRange{ops[0]});
1620 }
1621 case X86::BI__builtin_ia32_rdrand16_step:
1622 case X86::BI__builtin_ia32_rdrand32_step:
1623 case X86::BI__builtin_ia32_rdrand64_step:
1624 case X86::BI__builtin_ia32_rdseed16_step:
1625 case X86::BI__builtin_ia32_rdseed32_step:
1626 case X86::BI__builtin_ia32_rdseed64_step:
1627 case X86::BI__builtin_ia32_addcarryx_u32:
1628 case X86::BI__builtin_ia32_addcarryx_u64:
1629 case X86::BI__builtin_ia32_subborrow_u32:
1630 case X86::BI__builtin_ia32_subborrow_u64:
1631 case X86::BI__builtin_ia32_fpclassps128_mask:
1632 case X86::BI__builtin_ia32_fpclassps256_mask:
1633 case X86::BI__builtin_ia32_fpclassps512_mask:
1634 case X86::BI__builtin_ia32_vfpclassbf16128_mask:
1635 case X86::BI__builtin_ia32_vfpclassbf16256_mask:
1636 case X86::BI__builtin_ia32_vfpclassbf16512_mask:
1637 case X86::BI__builtin_ia32_fpclassph128_mask:
1638 case X86::BI__builtin_ia32_fpclassph256_mask:
1639 case X86::BI__builtin_ia32_fpclassph512_mask:
1640 case X86::BI__builtin_ia32_fpclasspd128_mask:
1641 case X86::BI__builtin_ia32_fpclasspd256_mask:
1642 case X86::BI__builtin_ia32_fpclasspd512_mask:
1643 case X86::BI__builtin_ia32_vp2intersect_q_512:
1644 case X86::BI__builtin_ia32_vp2intersect_q_256:
1645 case X86::BI__builtin_ia32_vp2intersect_q_128:
1646 case X86::BI__builtin_ia32_vp2intersect_d_512:
1647 case X86::BI__builtin_ia32_vp2intersect_d_256:
1648 case X86::BI__builtin_ia32_vp2intersect_d_128:
1649 case X86::BI__builtin_ia32_vpmultishiftqb128:
1650 case X86::BI__builtin_ia32_vpmultishiftqb256:
1651 case X86::BI__builtin_ia32_vpmultishiftqb512:
1652 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
1653 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
1654 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
1655 case X86::BI__builtin_ia32_cmpeqps:
1656 case X86::BI__builtin_ia32_cmpeqpd:
1657 case X86::BI__builtin_ia32_cmpltps:
1658 case X86::BI__builtin_ia32_cmpltpd:
1659 case X86::BI__builtin_ia32_cmpleps:
1660 case X86::BI__builtin_ia32_cmplepd:
1661 case X86::BI__builtin_ia32_cmpunordps:
1662 case X86::BI__builtin_ia32_cmpunordpd:
1663 case X86::BI__builtin_ia32_cmpneqps:
1664 case X86::BI__builtin_ia32_cmpneqpd:
1665 cgm.errorNYI(expr->getSourceRange(),
1666 std::string("unimplemented X86 builtin call: ") +
1667 getContext().BuiltinInfo.getName(builtinID));
1668 return mlir::Value{};
1669 case X86::BI__builtin_ia32_cmpnltps:
1670 case X86::BI__builtin_ia32_cmpnltpd:
1672 cir::CmpOpKind::lt, true);
1673 case X86::BI__builtin_ia32_cmpnleps:
1674 case X86::BI__builtin_ia32_cmpnlepd:
1676 cir::CmpOpKind::le, true);
1677 case X86::BI__builtin_ia32_cmpordps:
1678 case X86::BI__builtin_ia32_cmpordpd:
1679 case X86::BI__builtin_ia32_cmpph128_mask:
1680 case X86::BI__builtin_ia32_cmpph256_mask:
1681 case X86::BI__builtin_ia32_cmpph512_mask:
1682 case X86::BI__builtin_ia32_cmpps128_mask:
1683 case X86::BI__builtin_ia32_cmpps256_mask:
1684 case X86::BI__builtin_ia32_cmpps512_mask:
1685 case X86::BI__builtin_ia32_cmppd128_mask:
1686 case X86::BI__builtin_ia32_cmppd256_mask:
1687 case X86::BI__builtin_ia32_cmppd512_mask:
1688 case X86::BI__builtin_ia32_vcmpbf16512_mask:
1689 case X86::BI__builtin_ia32_vcmpbf16256_mask:
1690 case X86::BI__builtin_ia32_vcmpbf16128_mask:
1691 case X86::BI__builtin_ia32_cmpps:
1692 case X86::BI__builtin_ia32_cmpps256:
1693 case X86::BI__builtin_ia32_cmppd:
1694 case X86::BI__builtin_ia32_cmppd256:
1695 case X86::BI__builtin_ia32_cmpeqss:
1696 case X86::BI__builtin_ia32_cmpltss:
1697 case X86::BI__builtin_ia32_cmpless:
1698 case X86::BI__builtin_ia32_cmpunordss:
1699 case X86::BI__builtin_ia32_cmpneqss:
1700 case X86::BI__builtin_ia32_cmpnltss:
1701 case X86::BI__builtin_ia32_cmpnless:
1702 case X86::BI__builtin_ia32_cmpordss:
1703 case X86::BI__builtin_ia32_cmpeqsd:
1704 case X86::BI__builtin_ia32_cmpltsd:
1705 case X86::BI__builtin_ia32_cmplesd:
1706 case X86::BI__builtin_ia32_cmpunordsd:
1707 case X86::BI__builtin_ia32_cmpneqsd:
1708 case X86::BI__builtin_ia32_cmpnltsd:
1709 case X86::BI__builtin_ia32_cmpnlesd:
1710 case X86::BI__builtin_ia32_cmpordsd:
1711 case X86::BI__builtin_ia32_vcvtph2ps_mask:
1712 case X86::BI__builtin_ia32_vcvtph2ps256_mask:
1713 case X86::BI__builtin_ia32_vcvtph2ps512_mask:
1714 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask:
1715 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
1716 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
1717 case X86::BI__cpuid:
1718 case X86::BI__cpuidex:
1719 case X86::BI__emul:
1720 case X86::BI__emulu:
1721 case X86::BI__mulh:
1722 case X86::BI__umulh:
1723 case X86::BI_mul128:
1724 case X86::BI_umul128: {
1725 cgm.errorNYI(expr->getSourceRange(),
1726 std::string("unimplemented X86 builtin call: ") +
1727 getContext().BuiltinInfo.getName(builtinID));
1728 return mlir::Value{};
1729 }
1730 case X86::BI__faststorefence: {
1731 cir::AtomicFenceOp::create(
1732 builder, getLoc(expr->getExprLoc()),
1733 cir::MemOrder::SequentiallyConsistent,
1735 cir::SyncScopeKind::System));
1736 return mlir::Value{};
1737 }
1738 case X86::BI__shiftleft128:
1739 case X86::BI__shiftright128: {
1740 cgm.errorNYI(expr->getSourceRange(),
1741 std::string("unimplemented X86 builtin call: ") +
1742 getContext().BuiltinInfo.getName(builtinID));
1743 return mlir::Value{};
1744 }
1745 case X86::BI_ReadWriteBarrier:
1746 case X86::BI_ReadBarrier:
1747 case X86::BI_WriteBarrier: {
1748 cir::AtomicFenceOp::create(
1749 builder, getLoc(expr->getExprLoc()),
1750 cir::MemOrder::SequentiallyConsistent,
1752 cir::SyncScopeKind::SingleThread));
1753 return mlir::Value{};
1754 }
1755 case X86::BI_AddressOfReturnAddress:
1756 case X86::BI__stosb:
1757 case X86::BI__ud2:
1758 case X86::BI__int2c:
1759 case X86::BI__readfsbyte:
1760 case X86::BI__readfsword:
1761 case X86::BI__readfsdword:
1762 case X86::BI__readfsqword:
1763 case X86::BI__readgsbyte:
1764 case X86::BI__readgsword:
1765 case X86::BI__readgsdword:
1766 case X86::BI__readgsqword:
1767 case X86::BI__builtin_ia32_encodekey128_u32:
1768 case X86::BI__builtin_ia32_encodekey256_u32:
1769 case X86::BI__builtin_ia32_aesenc128kl_u8:
1770 case X86::BI__builtin_ia32_aesdec128kl_u8:
1771 case X86::BI__builtin_ia32_aesenc256kl_u8:
1772 case X86::BI__builtin_ia32_aesdec256kl_u8:
1773 case X86::BI__builtin_ia32_aesencwide128kl_u8:
1774 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
1775 case X86::BI__builtin_ia32_aesencwide256kl_u8:
1776 case X86::BI__builtin_ia32_aesdecwide256kl_u8:
1777 case X86::BI__builtin_ia32_vfcmaddcph512_mask:
1778 case X86::BI__builtin_ia32_vfmaddcph512_mask:
1779 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
1780 case X86::BI__builtin_ia32_vfmaddcsh_round_mask:
1781 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
1782 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3:
1783 case X86::BI__builtin_ia32_prefetchi:
1784 cgm.errorNYI(expr->getSourceRange(),
1785 std::string("unimplemented X86 builtin call: ") +
1786 getContext().BuiltinInfo.getName(builtinID));
1787 return mlir::Value{};
1788 }
1789}