CIRGenBuiltin.cpp Source File (original) (raw)

35 const CallExpr *e, mlir::Operation *calleeValue) {

38}

42 bool poisonZero = false) {

48 Op op;

49 if constexpr (std::is_same_v<Op, cir::BitClzOp> ||

50 std::is_same_v<Op, cir::BitCtzOp>)

52 else

55 mlir::Value result = op.getResult();

57 if (exprTy != result.getType())

61}

64 cir::SyncScopeKind syncScope) {

68 auto constOrdering = orderingVal.getDefiningOpcir::ConstantOp();

70 if (!constOrdering) {

73 cgf.cgm.errorNYI("Variable atomic fence ordering");

74 return;

75 }

77 auto constOrderingAttr = constOrdering.getValueAttrcir::IntAttr();

78 assert(constOrderingAttr && "Expected integer constant for ordering");

80 auto ordering = static_castcir::MemOrder\(constOrderingAttr.getUInt());

82 cir::AtomicFenceOp::create(

83 builder, cgf.getLoc(expr->getSourceRange()), ordering,

84 cir::SyncScopeKindAttr::get(&cgf.getMLIRContext(), syncScope));

85}

97 assert(type->isIntegerType() && "Given type is not an integer.");

98 unsigned width = type->isBooleanType() ? 1

101 bool isSigned = type->isSignedIntegerType();

102 return {width, isSigned};

103}

110 assert(types.size() > 0 && "Empty list of types.");

111

112

113 bool isSigned = llvm::any_of(types, [](const auto &t) { return t.isSigned; });

114

115

116

117

118

119 unsigned width = 0;

120 for (const auto &type : types)

121 width = std::max(width, type.width + (isSigned && type .isSigned));

122

123 return {width, isSigned};

124}

129

130

131

132

134

136 amount, isRotateLeft);

138}

144

147

148 auto call =

149 Operation::create(cgf.getBuilder(), arg.getLoc(), arg.getType(), arg);

151}

156 auto call =

157 Operation::create(cgf.getBuilder(), arg.getLoc(), arg.getType(), arg);

159}

162 unsigned builtinID) {

163

167 std::string("unimplemented X86 library function builtin call: ") +

169 } else {

171 std::string("unimplemented X86 builtin call: ") +

173 }

174

176}

179 unsigned builtinID) {

180 assert(builtinID == Builtin::BI__builtin_alloca ||

181 builtinID == Builtin::BI__builtin_alloca_uninitialized ||

182 builtinID == Builtin::BIalloca || builtinID == Builtin::BI_alloca);

183

184

186

187

189 const CharUnits suitableAlignmentInBytes =

191

192

193

194

195

196

197

199 mlir::Value allocaAddr = builder.createAlloca(

201 builder.getUInt8Ty(), "bi_alloca", suitableAlignmentInBytes, size);

202

203

204 if (builtinID != Builtin::BI__builtin_alloca_uninitialized) {

205

206

207

208

209 switch (cgf.getLangOpts().getTrivialAutoVarInit()) {

211

212 break;

215 cgf.cgm.errorNYI("trivial auto var init");

216 break;

217 }

218 }

219

220

221

222

223

224

225

230 "Non-default address space for alloca");

231 }

232

233

236}

242

243

244

251

252

253

254

255

258 }

259 }

260

262

264

265

266

267

268

269 unsigned builtinIDIfNoAsmLabel = fd->hasAttr() ? 0 : builtinID;

270

273

274 switch (builtinIDIfNoAsmLabel) {

275 default:

276 break;

277

278

279 case Builtin::BI__builtin_stdarg_start:

280 case Builtin::BI__builtin_va_start:

281 case Builtin::BI__va_start: {

282 mlir::Value vaList = builtinID == Builtin::BI__va_start

287 return {};

288 }

289

290 case Builtin::BI__builtin_va_end:

292 return {};

293 case Builtin::BI__builtin_va_copy: {

296 cir::VACopyOp::create(builder, dstPtr.getLoc(), dstPtr, srcPtr);

297 return {};

298 }

299 case Builtin::BIcos:

300 case Builtin::BIcosf:

301 case Builtin::BIcosl:

302 case Builtin::BI__builtin_cos:

303 case Builtin::BI__builtin_cosf:

304 case Builtin::BI__builtin_cosf16:

305 case Builtin::BI__builtin_cosl:

306 case Builtin::BI__builtin_cosf128:

309

310 case Builtin::BIceil:

311 case Builtin::BIceilf:

312 case Builtin::BIceill:

313 case Builtin::BI__builtin_ceil:

314 case Builtin::BI__builtin_ceilf:

315 case Builtin::BI__builtin_ceilf16:

316 case Builtin::BI__builtin_ceill:

317 case Builtin::BI__builtin_ceilf128:

320

321 case Builtin::BIexp:

322 case Builtin::BIexpf:

323 case Builtin::BIexpl:

324 case Builtin::BI__builtin_exp:

325 case Builtin::BI__builtin_expf:

326 case Builtin::BI__builtin_expf16:

327 case Builtin::BI__builtin_expl:

328 case Builtin::BI__builtin_expf128:

331

332 case Builtin::BIexp2:

333 case Builtin::BIexp2f:

334 case Builtin::BIexp2l:

335 case Builtin::BI__builtin_exp2:

336 case Builtin::BI__builtin_exp2f:

337 case Builtin::BI__builtin_exp2f16:

338 case Builtin::BI__builtin_exp2l:

339 case Builtin::BI__builtin_exp2f128:

342

343 case Builtin::BIfabs:

344 case Builtin::BIfabsf:

345 case Builtin::BIfabsl:

346 case Builtin::BI__builtin_fabs:

347 case Builtin::BI__builtin_fabsf:

348 case Builtin::BI__builtin_fabsf16:

349 case Builtin::BI__builtin_fabsl:

350 case Builtin::BI__builtin_fabsf128:

352

353 case Builtin::BIfloor:

354 case Builtin::BIfloorf:

355 case Builtin::BIfloorl:

356 case Builtin::BI__builtin_floor:

357 case Builtin::BI__builtin_floorf:

358 case Builtin::BI__builtin_floorf16:

359 case Builtin::BI__builtin_floorl:

360 case Builtin::BI__builtin_floorf128:

362

363 case Builtin::BI__assume:

364 case Builtin::BI__builtin_assume: {

367

369 cir::AssumeOp::create(builder, loc, argValue);

371 }

372

373 case Builtin::BI__builtin_assume_separate_storage: {

376 cir::AssumeSepStorageOp::create(builder, loc, value0, value1);

378 }

379

380 case Builtin::BI__builtin_assume_aligned: {

383 mlir::Value offsetValue =

385

386 std::optionalllvm::APSInt alignment =

388 assert(alignment.has_value() &&

389 "the second argument to __builtin_assume_aligned must be an "

390 "integral constant expression");

391

392 mlir::Value result =

394 alignment->getSExtValue(), offsetValue);

396 }

397

398 case Builtin::BI__builtin_complex: {

401 mlir::Value complex = builder.createComplexCreate(loc, real, imag);

403 }

404

405 case Builtin::BI__builtin_creal:

406 case Builtin::BI__builtin_crealf:

407 case Builtin::BI__builtin_creall:

408 case Builtin::BIcreal:

409 case Builtin::BIcrealf:

410 case Builtin::BIcreall: {

412 mlir::Value real = builder.createComplexReal(loc, complex);

414 }

415

416 case Builtin::BI__builtin_cimag:

417 case Builtin::BI__builtin_cimagf:

418 case Builtin::BI__builtin_cimagl:

419 case Builtin::BIcimag:

420 case Builtin::BIcimagf:

421 case Builtin::BIcimagl: {

423 mlir::Value imag = builder.createComplexImag(loc, complex);

425 }

426

427 case Builtin::BI__builtin_conj:

428 case Builtin::BI__builtin_conjf:

429 case Builtin::BI__builtin_conjl:

430 case Builtin::BIconj:

431 case Builtin::BIconjf:

432 case Builtin::BIconjl: {

435 cir::UnaryOpKind::Not, complex);

437 }

438

439 case Builtin::BI__builtin_clrsb:

440 case Builtin::BI__builtin_clrsbl:

441 case Builtin::BI__builtin_clrsbll:

443

444 case Builtin::BI__builtin_ctzs:

445 case Builtin::BI__builtin_ctz:

446 case Builtin::BI__builtin_ctzl:

447 case Builtin::BI__builtin_ctzll:

448 case Builtin::BI__builtin_ctzg:

451

452 case Builtin::BI__builtin_clzs:

453 case Builtin::BI__builtin_clz:

454 case Builtin::BI__builtin_clzl:

455 case Builtin::BI__builtin_clzll:

456 case Builtin::BI__builtin_clzg:

459

460 case Builtin::BI__builtin_ffs:

461 case Builtin::BI__builtin_ffsl:

462 case Builtin::BI__builtin_ffsll:

464

465 case Builtin::BI__builtin_parity:

466 case Builtin::BI__builtin_parityl:

467 case Builtin::BI__builtin_parityll:

469

470 case Builtin::BI__lzcnt16:

471 case Builtin::BI__lzcnt:

472 case Builtin::BI__lzcnt64:

475

476 case Builtin::BI__popcnt16:

477 case Builtin::BI__popcnt:

478 case Builtin::BI__popcnt64:

479 case Builtin::BI__builtin_popcount:

480 case Builtin::BI__builtin_popcountl:

481 case Builtin::BI__builtin_popcountll:

482 case Builtin::BI__builtin_popcountg:

484

485 case Builtin::BI__builtin_expect:

486 case Builtin::BI__builtin_expect_with_probability: {

489

490 mlir::FloatAttr probAttr;

491 if (builtinIDIfNoAsmLabel == Builtin::BI__builtin_expect_with_probability) {

492 llvm::APFloat probability(0.0);

494 [[maybe_unused]] bool evalSucceeded =

496 assert(evalSucceeded &&

497 "probability should be able to evaluate as float");

498 bool loseInfo = false;

499 probability.convert(llvm::APFloat::IEEEdouble(),

500 llvm::RoundingMode::Dynamic, &loseInfo);

501 probAttr = mlir::FloatAttr::get(mlir::Float64Type::get(&getMLIRContext()),

502 probability);

503 }

504

505 auto result = cir::ExpectOp::create(builder, loc, argValue.getType(),

506 argValue, expectedValue, probAttr);

508 }

509

510 case Builtin::BI__builtin_bswap16:

511 case Builtin::BI__builtin_bswap32:

512 case Builtin::BI__builtin_bswap64:

513 case Builtin::BI_byteswap_ushort:

514 case Builtin::BI_byteswap_ulong:

515 case Builtin::BI_byteswap_uint64: {

517 return RValue::get(cir::ByteSwapOp::create(builder, loc, arg));

518 }

519

520 case Builtin::BI__builtin_bitreverse8:

521 case Builtin::BI__builtin_bitreverse16:

522 case Builtin::BI__builtin_bitreverse32:

523 case Builtin::BI__builtin_bitreverse64: {

525 return RValue::get(cir::BitReverseOp::create(builder, loc, arg));

526 }

527

528 case Builtin::BI__builtin_rotateleft8:

529 case Builtin::BI__builtin_rotateleft16:

530 case Builtin::BI__builtin_rotateleft32:

531 case Builtin::BI__builtin_rotateleft64:

532 return emitRotate(e, true);

533

534 case Builtin::BI__builtin_rotateright8:

535 case Builtin::BI__builtin_rotateright16:

536 case Builtin::BI__builtin_rotateright32:

537 case Builtin::BI__builtin_rotateright64:

538 return emitRotate(e, false);

539

540 case Builtin::BI__builtin_coro_id:

541 case Builtin::BI__builtin_coro_promise:

542 case Builtin::BI__builtin_coro_resume:

543 case Builtin::BI__builtin_coro_noop:

544 case Builtin::BI__builtin_coro_destroy:

545 case Builtin::BI__builtin_coro_done:

546 case Builtin::BI__builtin_coro_alloc:

547 case Builtin::BI__builtin_coro_begin:

548 case Builtin::BI__builtin_coro_end:

549 case Builtin::BI__builtin_coro_suspend:

550 case Builtin::BI__builtin_coro_align:

551 cgm.errorNYI(e->getSourceRange(), "BI__builtin_coro_id like NYI");

553

554 case Builtin::BI__builtin_coro_frame: {

556 }

557 case Builtin::BI__builtin_coro_free:

558 case Builtin::BI__builtin_coro_size: {

560 mlir::Type ty = cgm.getTypes().getFunctionType(

561 cgm.getTypes().arrangeGlobalDeclaration(gd));

563 cir::FuncOp fnOp =

564 cgm.getOrCreateCIRFunction(nd->getName(), ty, gd, false);

565 fnOp.setBuiltin(true);

568 }

569

570 case Builtin::BI__builtin_constant_p: {

572

574 QualType argType = arg->getType();

575

576

579

580

583 mlir::castcir::IntType(resultType), 0));

584 }

585

586 if (arg->HasSideEffects(getContext())) {

587

588

591 mlir::castcir::IntType(resultType), 0));

592 }

593

597 "__builtin_constant_p: Obj-C object pointer");

598 return {};

599 }

600 argValue = builder.createBitcast(argValue, convertType(argType));

601

602 mlir::Value result = cir::IsConstantOp::create(

604

605 result = builder.createBoolToInt(result, resultType);

607 }

608 case Builtin::BI__builtin_dynamic_object_size:

609 case Builtin::BI__builtin_object_size: {

610 unsigned type =

613

614

615

616 bool isDynamic = builtinID == Builtin::BI__builtin_dynamic_object_size;

618 nullptr, isDynamic));

619 }

620

621 case Builtin::BI__builtin_prefetch: {

622 auto evaluateOperandAsInt = [&](const Expr *arg) {

624 [[maybe_unused]] bool evalSucceed =

625 arg->EvaluateAsInt(res, cgm.getASTContext());

626 assert(evalSucceed && "expression should be able to evaluate as int");

627 return res.Val.getInt().getZExtValue();

628 };

629

630 bool isWrite = false;

632 isWrite = evaluateOperandAsInt(e->getArg(1));

633

634 int locality = 3;

636 locality = evaluateOperandAsInt(e->getArg(2));

637

639 cir::PrefetchOp::create(builder, loc, address, locality, isWrite);

641 }

642 case Builtin::BI__builtin_readcyclecounter:

643 case Builtin::BI__builtin_readsteadycounter:

644 case Builtin::BI__builtin___clear_cache:

646 case Builtin::BI__builtin_trap:

647 emitTrap(loc, true);

649 case Builtin::BI__builtin_verbose_trap:

650 case Builtin::BI__debugbreak:

652 case Builtin::BI__builtin_unreachable:

655 case Builtin::BI__builtin_powi:

656 case Builtin::BI__builtin_powif:

657 case Builtin::BI__builtin_powil:

658 case Builtin::BI__builtin_frexpl:

659 case Builtin::BI__builtin_frexp:

660 case Builtin::BI__builtin_frexpf:

661 case Builtin::BI__builtin_frexpf128:

662 case Builtin::BI__builtin_frexpf16:

663 case Builtin::BImodf:

664 case Builtin::BImodff:

665 case Builtin::BImodfl:

666 case Builtin::BI__builtin_modf:

667 case Builtin::BI__builtin_modff:

668 case Builtin::BI__builtin_modfl:

669 case Builtin::BI__builtin_isgreater:

670 case Builtin::BI__builtin_isgreaterequal:

671 case Builtin::BI__builtin_isless:

672 case Builtin::BI__builtin_islessequal:

673 case Builtin::BI__builtin_islessgreater:

674 case Builtin::BI__builtin_isunordered:

675

676

677

678

679

680

681 case Builtin::BI__builtin_isnan: {

686 return RValue::get(builder.createBoolToInt(

687 builder.createIsFPClass(loc, v, cir::FPClassTest::Nan),

689 }

690

691 case Builtin::BI__builtin_issignaling: {

695 return RValue::get(builder.createBoolToInt(

696 builder.createIsFPClass(loc, v, cir::FPClassTest::SignalingNaN),

698 }

699

700 case Builtin::BI__builtin_isinf: {

705 return RValue::get(builder.createBoolToInt(

706 builder.createIsFPClass(loc, v, cir::FPClassTest::Infinity),

708 }

709 case Builtin::BIfinite:

710 case Builtin::BI__finite:

711 case Builtin::BIfinitef:

712 case Builtin::BI__finitef:

713 case Builtin::BIfinitel:

714 case Builtin::BI__finitel:

715 case Builtin::BI__builtin_isfinite: {

720 return RValue::get(builder.createBoolToInt(

721 builder.createIsFPClass(loc, v, cir::FPClassTest::Finite),

723 }

724

725 case Builtin::BI__builtin_isnormal: {

729 return RValue::get(builder.createBoolToInt(

730 builder.createIsFPClass(loc, v, cir::FPClassTest::Normal),

732 }

733

734 case Builtin::BI__builtin_issubnormal: {

738 return RValue::get(builder.createBoolToInt(

739 builder.createIsFPClass(loc, v, cir::FPClassTest::Subnormal),

741 }

742

743 case Builtin::BI__builtin_iszero: {

747 return RValue::get(builder.createBoolToInt(

748 builder.createIsFPClass(loc, v, cir::FPClassTest::Zero),

750 }

751 case Builtin::BI__builtin_isfpclass: {

754 break;

755

758 uint64_t test = result.Val.getInt().getLimitedValue();

760

761 return RValue::get(builder.createBoolToInt(

762 builder.createIsFPClass(loc, v, cir::FPClassTest(test)),

764 }

765 case Builtin::BI__builtin_nondeterministic_value:

766 case Builtin::BI__builtin_elementwise_abs:

768 case Builtin::BI__builtin_elementwise_acos:

770 case Builtin::BI__builtin_elementwise_asin:

772 case Builtin::BI__builtin_elementwise_atan:

774 case Builtin::BI__builtin_elementwise_atan2:

775 case Builtin::BI__builtin_elementwise_ceil:

776 case Builtin::BI__builtin_elementwise_exp:

777 case Builtin::BI__builtin_elementwise_exp2:

778 case Builtin::BI__builtin_elementwise_exp10:

779 case Builtin::BI__builtin_elementwise_ldexp:

780 case Builtin::BI__builtin_elementwise_log:

781 case Builtin::BI__builtin_elementwise_log2:

782 case Builtin::BI__builtin_elementwise_log10:

783 case Builtin::BI__builtin_elementwise_pow:

784 case Builtin::BI__builtin_elementwise_bitreverse:

786 case Builtin::BI__builtin_elementwise_cos:

788 case Builtin::BI__builtin_elementwise_cosh:

789 case Builtin::BI__builtin_elementwise_floor:

790 case Builtin::BI__builtin_elementwise_popcount:

791 case Builtin::BI__builtin_elementwise_roundeven:

792 case Builtin::BI__builtin_elementwise_round:

793 case Builtin::BI__builtin_elementwise_rint:

794 case Builtin::BI__builtin_elementwise_nearbyint:

795 case Builtin::BI__builtin_elementwise_sin:

796 case Builtin::BI__builtin_elementwise_sinh:

797 case Builtin::BI__builtin_elementwise_tan:

798 case Builtin::BI__builtin_elementwise_tanh:

799 case Builtin::BI__builtin_elementwise_trunc:

800 case Builtin::BI__builtin_elementwise_canonicalize:

801 case Builtin::BI__builtin_elementwise_copysign:

802 case Builtin::BI__builtin_elementwise_fma:

803 case Builtin::BI__builtin_elementwise_fshl:

804 case Builtin::BI__builtin_elementwise_fshr:

805 case Builtin::BI__builtin_elementwise_add_sat:

806 case Builtin::BI__builtin_elementwise_sub_sat:

807 case Builtin::BI__builtin_elementwise_max:

808 case Builtin::BI__builtin_elementwise_min:

809 case Builtin::BI__builtin_elementwise_maxnum:

810 case Builtin::BI__builtin_elementwise_minnum:

811 case Builtin::BI__builtin_elementwise_maximum:

812 case Builtin::BI__builtin_elementwise_minimum:

813 case Builtin::BI__builtin_elementwise_maximumnum:

814 case Builtin::BI__builtin_elementwise_minimumnum:

815 case Builtin::BI__builtin_reduce_max:

816 case Builtin::BI__builtin_reduce_min:

817 case Builtin::BI__builtin_reduce_add:

818 case Builtin::BI__builtin_reduce_mul:

819 case Builtin::BI__builtin_reduce_xor:

820 case Builtin::BI__builtin_reduce_or:

821 case Builtin::BI__builtin_reduce_and:

822 case Builtin::BI__builtin_reduce_maximum:

823 case Builtin::BI__builtin_reduce_minimum:

824 case Builtin::BI__builtin_matrix_transpose:

825 case Builtin::BI__builtin_matrix_column_major_load:

826 case Builtin::BI__builtin_matrix_column_major_store:

827 case Builtin::BI__builtin_masked_load:

828 case Builtin::BI__builtin_masked_expand_load:

829 case Builtin::BI__builtin_masked_gather:

830 case Builtin::BI__builtin_masked_store:

831 case Builtin::BI__builtin_masked_compress_store:

832 case Builtin::BI__builtin_masked_scatter:

833 case Builtin::BI__builtin_isinf_sign:

834 case Builtin::BI__builtin_flt_rounds:

835 case Builtin::BI__builtin_set_flt_rounds:

836 case Builtin::BI__builtin_fpclassify:

838 case Builtin::BIalloca:

839 case Builtin::BI_alloca:

840 case Builtin::BI__builtin_alloca_uninitialized:

841 case Builtin::BI__builtin_alloca:

843 case Builtin::BI__builtin_alloca_with_align_uninitialized:

844 case Builtin::BI__builtin_alloca_with_align:

845 case Builtin::BI__builtin_infer_alloc_token:

846 case Builtin::BIbzero:

847 case Builtin::BI__builtin_bzero:

848 case Builtin::BIbcopy:

849 case Builtin::BI__builtin_bcopy:

851 case Builtin::BImemcpy:

852 case Builtin::BI__builtin_memcpy:

853 case Builtin::BImempcpy:

854 case Builtin::BI__builtin_mempcpy:

855 case Builtin::BI__builtin_memcpy_inline:

856 case Builtin::BI__builtin_char_memchr:

857 case Builtin::BI__builtin___memcpy_chk:

858 case Builtin::BI__builtin_objc_memmove_collectable:

859 case Builtin::BI__builtin___memmove_chk:

860 case Builtin::BI__builtin_trivially_relocate:

861 case Builtin::BImemmove:

862 case Builtin::BI__builtin_memmove:

863 case Builtin::BImemset:

864 case Builtin::BI__builtin_memset:

865 case Builtin::BI__builtin_memset_inline:

866 case Builtin::BI__builtin___memset_chk:

867 case Builtin::BI__builtin_wmemchr:

868 case Builtin::BI__builtin_wmemcmp:

869 break;

870 case Builtin::BI__builtin_dwarf_cfa:

872 case Builtin::BI__builtin_return_address:

873 case Builtin::BI_ReturnAddress:

874 case Builtin::BI__builtin_frame_address: {

877 if (builtinID == Builtin::BI__builtin_return_address) {

878 return RValue::get(cir::ReturnAddrOp::create(

879 builder, loc,

880 builder.getConstAPInt(loc, builder.getUInt32Ty(), level)));

881 }

882 return RValue::get(cir::FrameAddrOp::create(

883 builder, loc,

884 builder.getConstAPInt(loc, builder.getUInt32Ty(), level)));

885 }

886 case Builtin::BI__builtin_extract_return_addr:

887 case Builtin::BI__builtin_frob_return_addr:

888 case Builtin::BI__builtin_dwarf_sp_column:

889 case Builtin::BI__builtin_init_dwarf_reg_size_table:

890 case Builtin::BI__builtin_eh_return:

891 case Builtin::BI__builtin_unwind_init:

892 case Builtin::BI__builtin_extend_pointer:

893 case Builtin::BI__builtin_setjmp:

894 case Builtin::BI__builtin_longjmp:

895 case Builtin::BI__builtin_launder:

896 case Builtin::BI__sync_fetch_and_add:

897 case Builtin::BI__sync_fetch_and_sub:

898 case Builtin::BI__sync_fetch_and_or:

899 case Builtin::BI__sync_fetch_and_and:

900 case Builtin::BI__sync_fetch_and_xor:

901 case Builtin::BI__sync_fetch_and_nand:

902 case Builtin::BI__sync_add_and_fetch:

903 case Builtin::BI__sync_sub_and_fetch:

904 case Builtin::BI__sync_and_and_fetch:

905 case Builtin::BI__sync_or_and_fetch:

906 case Builtin::BI__sync_xor_and_fetch:

907 case Builtin::BI__sync_nand_and_fetch:

908 case Builtin::BI__sync_val_compare_and_swap:

909 case Builtin::BI__sync_bool_compare_and_swap:

910 case Builtin::BI__sync_lock_test_and_set:

911 case Builtin::BI__sync_lock_release:

912 case Builtin::BI__sync_swap:

913 case Builtin::BI__sync_fetch_and_add_1:

914 case Builtin::BI__sync_fetch_and_add_2:

915 case Builtin::BI__sync_fetch_and_add_4:

916 case Builtin::BI__sync_fetch_and_add_8:

917 case Builtin::BI__sync_fetch_and_add_16:

918 case Builtin::BI__sync_fetch_and_sub_1:

919 case Builtin::BI__sync_fetch_and_sub_2:

920 case Builtin::BI__sync_fetch_and_sub_4:

921 case Builtin::BI__sync_fetch_and_sub_8:

922 case Builtin::BI__sync_fetch_and_sub_16:

923 case Builtin::BI__sync_fetch_and_or_1:

924 case Builtin::BI__sync_fetch_and_or_2:

925 case Builtin::BI__sync_fetch_and_or_4:

926 case Builtin::BI__sync_fetch_and_or_8:

927 case Builtin::BI__sync_fetch_and_or_16:

928 case Builtin::BI__sync_fetch_and_and_1:

929 case Builtin::BI__sync_fetch_and_and_2:

930 case Builtin::BI__sync_fetch_and_and_4:

931 case Builtin::BI__sync_fetch_and_and_8:

932 case Builtin::BI__sync_fetch_and_and_16:

933 case Builtin::BI__sync_fetch_and_xor_1:

934 case Builtin::BI__sync_fetch_and_xor_2:

935 case Builtin::BI__sync_fetch_and_xor_4:

936 case Builtin::BI__sync_fetch_and_xor_8:

937 case Builtin::BI__sync_fetch_and_xor_16:

938 case Builtin::BI__sync_fetch_and_nand_1:

939 case Builtin::BI__sync_fetch_and_nand_2:

940 case Builtin::BI__sync_fetch_and_nand_4:

941 case Builtin::BI__sync_fetch_and_nand_8:

942 case Builtin::BI__sync_fetch_and_nand_16:

943 case Builtin::BI__sync_fetch_and_min:

944 case Builtin::BI__sync_fetch_and_max:

945 case Builtin::BI__sync_fetch_and_umin:

946 case Builtin::BI__sync_fetch_and_umax:

947 case Builtin::BI__sync_add_and_fetch_1:

948 case Builtin::BI__sync_add_and_fetch_2:

949 case Builtin::BI__sync_add_and_fetch_4:

950 case Builtin::BI__sync_add_and_fetch_8:

951 case Builtin::BI__sync_add_and_fetch_16:

952 case Builtin::BI__sync_sub_and_fetch_1:

953 case Builtin::BI__sync_sub_and_fetch_2:

954 case Builtin::BI__sync_sub_and_fetch_4:

955 case Builtin::BI__sync_sub_and_fetch_8:

956 case Builtin::BI__sync_sub_and_fetch_16:

957 case Builtin::BI__sync_and_and_fetch_1:

958 case Builtin::BI__sync_and_and_fetch_2:

959 case Builtin::BI__sync_and_and_fetch_4:

960 case Builtin::BI__sync_and_and_fetch_8:

961 case Builtin::BI__sync_and_and_fetch_16:

962 case Builtin::BI__sync_or_and_fetch_1:

963 case Builtin::BI__sync_or_and_fetch_2:

964 case Builtin::BI__sync_or_and_fetch_4:

965 case Builtin::BI__sync_or_and_fetch_8:

966 case Builtin::BI__sync_or_and_fetch_16:

967 case Builtin::BI__sync_xor_and_fetch_1:

968 case Builtin::BI__sync_xor_and_fetch_2:

969 case Builtin::BI__sync_xor_and_fetch_4:

970 case Builtin::BI__sync_xor_and_fetch_8:

971 case Builtin::BI__sync_xor_and_fetch_16:

972 case Builtin::BI__sync_nand_and_fetch_1:

973 case Builtin::BI__sync_nand_and_fetch_2:

974 case Builtin::BI__sync_nand_and_fetch_4:

975 case Builtin::BI__sync_nand_and_fetch_8:

976 case Builtin::BI__sync_nand_and_fetch_16:

977 case Builtin::BI__sync_val_compare_and_swap_1:

978 case Builtin::BI__sync_val_compare_and_swap_2:

979 case Builtin::BI__sync_val_compare_and_swap_4:

980 case Builtin::BI__sync_val_compare_and_swap_8:

981 case Builtin::BI__sync_val_compare_and_swap_16:

982 case Builtin::BI__sync_bool_compare_and_swap_1:

983 case Builtin::BI__sync_bool_compare_and_swap_2:

984 case Builtin::BI__sync_bool_compare_and_swap_4:

985 case Builtin::BI__sync_bool_compare_and_swap_8:

986 case Builtin::BI__sync_bool_compare_and_swap_16:

987 case Builtin::BI__sync_swap_1:

988 case Builtin::BI__sync_swap_2:

989 case Builtin::BI__sync_swap_4:

990 case Builtin::BI__sync_swap_8:

991 case Builtin::BI__sync_swap_16:

992 case Builtin::BI__sync_lock_test_and_set_1:

993 case Builtin::BI__sync_lock_test_and_set_2:

994 case Builtin::BI__sync_lock_test_and_set_4:

995 case Builtin::BI__sync_lock_test_and_set_8:

996 case Builtin::BI__sync_lock_test_and_set_16:

997 case Builtin::BI__sync_lock_release_1:

998 case Builtin::BI__sync_lock_release_2:

999 case Builtin::BI__sync_lock_release_4:

1000 case Builtin::BI__sync_lock_release_8:

1001 case Builtin::BI__sync_lock_release_16:

1002 case Builtin::BI__sync_synchronize:

1003 case Builtin::BI__builtin_nontemporal_load:

1004 case Builtin::BI__builtin_nontemporal_store:

1005 case Builtin::BI__c11_atomic_is_lock_free:

1006 case Builtin::BI__atomic_is_lock_free:

1007 case Builtin::BI__atomic_test_and_set:

1008 case Builtin::BI__atomic_clear:

1010 case Builtin::BI__atomic_thread_fence: {

1013 }

1014 case Builtin::BI__atomic_signal_fence: {

1017 }

1018 case Builtin::BI__c11_atomic_thread_fence:

1019 case Builtin::BI__c11_atomic_signal_fence:

1020 case Builtin::BI__scoped_atomic_thread_fence:

1021 case Builtin::BI__builtin_signbit:

1022 case Builtin::BI__builtin_signbitf:

1023 case Builtin::BI__builtin_signbitl:

1024 case Builtin::BI__warn_memset_zero_len:

1025 case Builtin::BI__annotation:

1026 case Builtin::BI__builtin_annotation:

1027 case Builtin::BI__builtin_addcb:

1028 case Builtin::BI__builtin_addcs:

1029 case Builtin::BI__builtin_addc:

1030 case Builtin::BI__builtin_addcl:

1031 case Builtin::BI__builtin_addcll:

1032 case Builtin::BI__builtin_subcb:

1033 case Builtin::BI__builtin_subcs:

1034 case Builtin::BI__builtin_subc:

1035 case Builtin::BI__builtin_subcl:

1036 case Builtin::BI__builtin_subcll:

1038

1039 case Builtin::BI__builtin_add_overflow:

1040 case Builtin::BI__builtin_sub_overflow:

1041 case Builtin::BI__builtin_mul_overflow: {

1045

1048

1049 WidthAndSignedness leftInfo =

1051 WidthAndSignedness rightInfo =

1053 WidthAndSignedness resultInfo =

1055

1056

1057

1058

1059 WidthAndSignedness encompassingInfo =

1061

1062 auto encompassingCIRTy = cir::IntType::get(

1063 &getMLIRContext(), encompassingInfo.width, encompassingInfo.isSigned);

1064 auto resultCIRTy = mlir::castcir::IntType(cgm.convertType(resultQTy));

1065

1069

1070

1071 if (left.getType() != encompassingCIRTy)

1072 left =

1073 builder.createCast(cir::CastKind::integral, left, encompassingCIRTy);

1074 if (right.getType() != encompassingCIRTy)

1075 right =

1076 builder.createCast(cir::CastKind::integral, right, encompassingCIRTy);

1077

1078

1079 cir::BinOpOverflowKind opKind;

1080 switch (builtinID) {

1081 default:

1082 llvm_unreachable("Unknown overflow builtin id.");

1083 case Builtin::BI__builtin_add_overflow:

1084 opKind = cir::BinOpOverflowKind::Add;

1085 break;

1086 case Builtin::BI__builtin_sub_overflow:

1087 opKind = cir::BinOpOverflowKind::Sub;

1088 break;

1089 case Builtin::BI__builtin_mul_overflow:

1090 opKind = cir::BinOpOverflowKind::Mul;

1091 break;

1092 }

1093

1095 auto arithOp = cir::BinOpOverflowOp::create(builder, loc, resultCIRTy,

1096 opKind, left, right);

1097

1098

1099

1100

1101

1102

1103

1104

1105

1106

1107

1108 bool isVolatile =

1110 builder.createStore(loc, emitToMemory(arithOp.getResult(), resultQTy),

1111 resultPtr, isVolatile);

1112

1113 return RValue::get(arithOp.getOverflow());

1114 }

1115

1116 case Builtin::BI__builtin_uadd_overflow:

1117 case Builtin::BI__builtin_uaddl_overflow:

1118 case Builtin::BI__builtin_uaddll_overflow:

1119 case Builtin::BI__builtin_usub_overflow:

1120 case Builtin::BI__builtin_usubl_overflow:

1121 case Builtin::BI__builtin_usubll_overflow:

1122 case Builtin::BI__builtin_umul_overflow:

1123 case Builtin::BI__builtin_umull_overflow:

1124 case Builtin::BI__builtin_umulll_overflow:

1125 case Builtin::BI__builtin_sadd_overflow:

1126 case Builtin::BI__builtin_saddl_overflow:

1127 case Builtin::BI__builtin_saddll_overflow:

1128 case Builtin::BI__builtin_ssub_overflow:

1129 case Builtin::BI__builtin_ssubl_overflow:

1130 case Builtin::BI__builtin_ssubll_overflow:

1131 case Builtin::BI__builtin_smul_overflow:

1132 case Builtin::BI__builtin_smull_overflow:

1133 case Builtin::BI__builtin_smulll_overflow: {

1134

1137

1140

1141

1142 cir::BinOpOverflowKind arithKind;

1143 switch (builtinID) {

1144 default:

1145 llvm_unreachable("Unknown overflow builtin id.");

1146 case Builtin::BI__builtin_uadd_overflow:

1147 case Builtin::BI__builtin_uaddl_overflow:

1148 case Builtin::BI__builtin_uaddll_overflow:

1149 case Builtin::BI__builtin_sadd_overflow:

1150 case Builtin::BI__builtin_saddl_overflow:

1151 case Builtin::BI__builtin_saddll_overflow:

1152 arithKind = cir::BinOpOverflowKind::Add;

1153 break;

1154 case Builtin::BI__builtin_usub_overflow:

1155 case Builtin::BI__builtin_usubl_overflow:

1156 case Builtin::BI__builtin_usubll_overflow:

1157 case Builtin::BI__builtin_ssub_overflow:

1158 case Builtin::BI__builtin_ssubl_overflow:

1159 case Builtin::BI__builtin_ssubll_overflow:

1160 arithKind = cir::BinOpOverflowKind::Sub;

1161 break;

1162 case Builtin::BI__builtin_umul_overflow:

1163 case Builtin::BI__builtin_umull_overflow:

1164 case Builtin::BI__builtin_umulll_overflow:

1165 case Builtin::BI__builtin_smul_overflow:

1166 case Builtin::BI__builtin_smull_overflow:

1167 case Builtin::BI__builtin_smulll_overflow:

1168 arithKind = cir::BinOpOverflowKind::Mul;

1169 break;

1170 }

1171

1174 auto resultCIRTy = mlir::castcir::IntType(cgm.convertType(resultQTy));

1175

1177 cir::BinOpOverflowOp arithOp = cir::BinOpOverflowOp::create(

1178 builder, loc, resultCIRTy, arithKind, x, y);

1179

1180 bool isVolatile =

1182 builder.createStore(loc, emitToMemory(arithOp.getResult(), resultQTy),

1183 resultPtr, isVolatile);

1184

1185 return RValue::get(arithOp.getOverflow());

1186 }

1187

1188 case Builtin::BIaddressof:

1189 case Builtin::BI__addressof:

1190 case Builtin::BI__builtin_addressof:

1191 case Builtin::BI__builtin_function_start:

1193 case Builtin::BI__builtin_operator_new:

1196 case Builtin::BI__builtin_operator_delete:

1200 case Builtin::BI__builtin_is_aligned:

1201 case Builtin::BI__builtin_align_up:

1202 case Builtin::BI__builtin_align_down:

1203 case Builtin::BI__noop:

1204 case Builtin::BI__builtin_call_with_static_chain:

1205 case Builtin::BI_InterlockedExchange8:

1206 case Builtin::BI_InterlockedExchange16:

1207 case Builtin::BI_InterlockedExchange:

1208 case Builtin::BI_InterlockedExchangePointer:

1209 case Builtin::BI_InterlockedCompareExchangePointer:

1210 case Builtin::BI_InterlockedCompareExchangePointer_nf:

1211 case Builtin::BI_InterlockedCompareExchange8:

1212 case Builtin::BI_InterlockedCompareExchange16:

1213 case Builtin::BI_InterlockedCompareExchange:

1214 case Builtin::BI_InterlockedCompareExchange64:

1215 case Builtin::BI_InterlockedIncrement16:

1216 case Builtin::BI_InterlockedIncrement:

1217 case Builtin::BI_InterlockedDecrement16:

1218 case Builtin::BI_InterlockedDecrement:

1219 case Builtin::BI_InterlockedAnd8:

1220 case Builtin::BI_InterlockedAnd16:

1221 case Builtin::BI_InterlockedAnd:

1222 case Builtin::BI_InterlockedExchangeAdd8:

1223 case Builtin::BI_InterlockedExchangeAdd16:

1224 case Builtin::BI_InterlockedExchangeAdd:

1225 case Builtin::BI_InterlockedExchangeSub8:

1226 case Builtin::BI_InterlockedExchangeSub16:

1227 case Builtin::BI_InterlockedExchangeSub:

1228 case Builtin::BI_InterlockedOr8:

1229 case Builtin::BI_InterlockedOr16:

1230 case Builtin::BI_InterlockedOr:

1231 case Builtin::BI_InterlockedXor8:

1232 case Builtin::BI_InterlockedXor16:

1233 case Builtin::BI_InterlockedXor:

1234 case Builtin::BI_bittest64:

1235 case Builtin::BI_bittest:

1236 case Builtin::BI_bittestandcomplement64:

1237 case Builtin::BI_bittestandcomplement:

1238 case Builtin::BI_bittestandreset64:

1239 case Builtin::BI_bittestandreset:

1240 case Builtin::BI_bittestandset64:

1241 case Builtin::BI_bittestandset:

1242 case Builtin::BI_interlockedbittestandreset:

1243 case Builtin::BI_interlockedbittestandreset64:

1244 case Builtin::BI_interlockedbittestandreset64_acq:

1245 case Builtin::BI_interlockedbittestandreset64_rel:

1246 case Builtin::BI_interlockedbittestandreset64_nf:

1247 case Builtin::BI_interlockedbittestandset64:

1248 case Builtin::BI_interlockedbittestandset64_acq:

1249 case Builtin::BI_interlockedbittestandset64_rel:

1250 case Builtin::BI_interlockedbittestandset64_nf:

1251 case Builtin::BI_interlockedbittestandset:

1252 case Builtin::BI_interlockedbittestandset_acq:

1253 case Builtin::BI_interlockedbittestandset_rel:

1254 case Builtin::BI_interlockedbittestandset_nf:

1255 case Builtin::BI_interlockedbittestandreset_acq:

1256 case Builtin::BI_interlockedbittestandreset_rel:

1257 case Builtin::BI_interlockedbittestandreset_nf:

1258 case Builtin::BI__iso_volatile_load8:

1259 case Builtin::BI__iso_volatile_load16:

1260 case Builtin::BI__iso_volatile_load32:

1261 case Builtin::BI__iso_volatile_load64:

1262 case Builtin::BI__iso_volatile_store8:

1263 case Builtin::BI__iso_volatile_store16:

1264 case Builtin::BI__iso_volatile_store32:

1265 case Builtin::BI__iso_volatile_store64:

1266 case Builtin::BI__builtin_ptrauth_sign_constant:

1267 case Builtin::BI__builtin_ptrauth_auth:

1268 case Builtin::BI__builtin_ptrauth_auth_and_resign:

1269 case Builtin::BI__builtin_ptrauth_blend_discriminator:

1270 case Builtin::BI__builtin_ptrauth_sign_generic_data:

1271 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:

1272 case Builtin::BI__builtin_ptrauth_strip:

1273 case Builtin::BI__builtin_get_vtable_pointer:

1274 case Builtin::BI__exception_code:

1275 case Builtin::BI_exception_code:

1276 case Builtin::BI__exception_info:

1277 case Builtin::BI_exception_info:

1278 case Builtin::BI__abnormal_termination:

1279 case Builtin::BI_abnormal_termination:

1280 case Builtin::BI_setjmpex:

1281 case Builtin::BI_setjmp:

1282 case Builtin::BImove:

1283 case Builtin::BImove_if_noexcept:

1284 case Builtin::BIforward:

1285 case Builtin::BIforward_like:

1286 case Builtin::BIas_const:

1287 case Builtin::BI__GetExceptionInfo:

1288 case Builtin::BI__fastfail:

1289 case Builtin::BIread_pipe:

1290 case Builtin::BIwrite_pipe:

1291 case Builtin::BIreserve_read_pipe:

1292 case Builtin::BIreserve_write_pipe:

1293 case Builtin::BIwork_group_reserve_read_pipe:

1294 case Builtin::BIwork_group_reserve_write_pipe:

1295 case Builtin::BIsub_group_reserve_read_pipe:

1296 case Builtin::BIsub_group_reserve_write_pipe:

1297 case Builtin::BIcommit_read_pipe:

1298 case Builtin::BIcommit_write_pipe:

1299 case Builtin::BIwork_group_commit_read_pipe:

1300 case Builtin::BIwork_group_commit_write_pipe:

1301 case Builtin::BIsub_group_commit_read_pipe:

1302 case Builtin::BIsub_group_commit_write_pipe:

1303 case Builtin::BIget_pipe_num_packets:

1304 case Builtin::BIget_pipe_max_packets:

1305 case Builtin::BIto_global:

1306 case Builtin::BIto_local:

1307 case Builtin::BIto_private:

1308 case Builtin::BIenqueue_kernel:

1309 case Builtin::BIget_kernel_work_group_size:

1310 case Builtin::BIget_kernel_preferred_work_group_size_multiple:

1311 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:

1312 case Builtin::BIget_kernel_sub_group_count_for_ndrange:

1313 case Builtin::BI__builtin_store_half:

1314 case Builtin::BI__builtin_store_halff:

1315 case Builtin::BI__builtin_load_half:

1316 case Builtin::BI__builtin_load_halff:

1318 case Builtin::BI__builtin_printf:

1319 case Builtin::BIprintf:

1320 break;

1321 case Builtin::BI__builtin_canonicalize:

1322 case Builtin::BI__builtin_canonicalizef:

1323 case Builtin::BI__builtin_canonicalizef16:

1324 case Builtin::BI__builtin_canonicalizel:

1325 case Builtin::BI__builtin_thread_pointer:

1326 case Builtin::BI__builtin_os_log_format:

1327 case Builtin::BI__xray_customevent:

1328 case Builtin::BI__xray_typedevent:

1329 case Builtin::BI__builtin_ms_va_start:

1330 case Builtin::BI__builtin_ms_va_end:

1331 case Builtin::BI__builtin_ms_va_copy:

1332 case Builtin::BI__builtin_get_device_side_mangled_name:

1334 }

1335

1336

1337

1338

1339 if (getContext().BuiltinInfo.isLibFunction(builtinID))

1341 cgm.getBuiltinLibFunction(fd, builtinID));

1342

1343

1344

1345

1346

1349 cgm.errorNYI(e->getSourceRange(), "aggregate return value from builtin");

1351 }

1352

1353

1354

1355

1356

1357

1358

1359 if (std::optionalmlir::Value rst =

1361 mlir::Value v = rst.value();

1362

1363

1364 if (!v)

1366

1367 switch (evalKind) {

1369 if (mlir::isacir::VoidType(v.getType()))

1373 cgm.errorNYI(e->getSourceRange(), "aggregate return value from builtin");

1376 llvm_unreachable("No current target builtin returns complex");

1377 }

1378 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");

1379 }

1380

1382 std::string("unimplemented builtin call: ") +

1383 getContext().BuiltinInfo.getName(builtinID));

1385}

1390 llvm::Triple::ArchType arch) {

1391

1392

1393

1394

1395

1396

1399 return std::nullopt;

1400

1401 switch (arch) {

1402 case llvm::Triple::arm:

1403 case llvm::Triple::armeb:

1404 case llvm::Triple::thumb:

1405 case llvm::Triple::thumbeb:

1406

1407

1408 return std::nullopt;

1409 case llvm::Triple::aarch64:

1410 case llvm::Triple::aarch64_32:

1411 case llvm::Triple::aarch64_be:

1413 case llvm::Triple::bpfeb:

1414 case llvm::Triple::bpfel:

1415

1416

1417 return std::nullopt;

1418

1419 case llvm::Triple::x86:

1420 case llvm::Triple::x86_64:

1422

1423 case llvm::Triple::ppc:

1424 case llvm::Triple::ppcle:

1425 case llvm::Triple::ppc64:

1426 case llvm::Triple::ppc64le:

1427 case llvm::Triple::r600:

1428 case llvm::Triple::amdgcn:

1429 case llvm::Triple::systemz:

1430 case llvm::Triple::nvptx:

1431 case llvm::Triple::nvptx64:

1432 case llvm::Triple::wasm32:

1433 case llvm::Triple::wasm64:

1434 case llvm::Triple::hexagon:

1435 case llvm::Triple::riscv32:

1436 case llvm::Triple::riscv64:

1437

1438

1439 return std::nullopt;

1440 default:

1441 return std::nullopt;

1442 }

1443}

1448 if (getContext().BuiltinInfo.isAuxBuiltinID(builtinID)) {

1449 assert(getContext().getAuxTargetInfo() && "Missing aux target info");

1451 this, getContext().BuiltinInfo.getAuxBuiltinID(builtinID), e,

1453 }

1454

1456 getTarget().getTriple().getArch());

1457}

1460 const unsigned iceArguments, const unsigned idx, const Expr *argExpr) {

1461 mlir::Value arg = {};

1462 if ((iceArguments & (1 << idx)) == 0) {

1464 } else {

1465

1466

1467 const std::optionalllvm::APSInt result =

1469 assert(result && "Expected argument to be a constant");

1471 }

1472 return arg;

1473}

1478 unsigned builtinID) {

1479 assert(astContext.BuiltinInfo.isLibFunction(builtinID));

1480

1481

1482

1483

1485

1487 name = astContext.BuiltinInfo.getName(builtinID).substr(10);

1488

1492}

1496 if ( sanOpts .has(SanitizerKind::Builtin))

1497 return argValue;

1498

1501 "emitCheckedArgForAssume: sanitizers are NYI");

1502 return {};

1503}

1506

1507

1508 cir::VAStartOp::create(builder, vaList.getLoc(), vaList, count);

1509}

1512 cir::VAEndOp::create(builder, vaList.getLoc(), vaList);

1513}

1521 mlir::Location loc = cgm.getLoc(ve->getExprLoc());

1524 return cir::VAArgOp::create(builder, loc, type, vaList);

1525}

1528 cir::IntType resType,

1529 mlir::Value emittedE,

1530 bool isDynamic) {

1532

1533

1534

1535

1538 (type & 2) ? 0 : -1);

1539

1540 mlir::Value ptr = emittedE ? emittedE : emitScalarExpr(e);

1541 assert(mlir::isacir::PointerType(ptr.getType()) &&

1542 "Non-pointer passed to __builtin_object_size?");

1543

1545

1546

1547

1548

1549 const bool min = ((type & 2) != 0);

1550

1551 auto op =

1553 min, true, isDynamic);

1554 return op.getResult();

1555}

1558 const Expr *e, unsigned type, cir::IntType resType, mlir::Value emittedE,

1559 bool isDynamic) {

1560 uint64_t objectSize;

1563 return builder.getConstInt(getLoc(e->getSourceRange()), resType, objectSize);

1564}