LLVM: lib/IR/AutoUpgrade.cpp Source File (original) (raw)

1

2

3

4

5

6

7

8

9

10

11

12

13

14

34#include "llvm/IR/IntrinsicsAArch64.h"

35#include "llvm/IR/IntrinsicsARM.h"

36#include "llvm/IR/IntrinsicsNVPTX.h"

37#include "llvm/IR/IntrinsicsRISCV.h"

38#include "llvm/IR/IntrinsicsWebAssembly.h"

39#include "llvm/IR/IntrinsicsX86.h"

53#include

54#include

55#include

56

57using namespace llvm;

58

61 cl::desc("Disable autoupgrade of debug info"));

62

64

65

66

69

70

71 Type *Arg0Type = F->getFunctionType()->getParamType(0);

73 return false;

74

75

78 return true;

79}

80

81

82

85

86 Type *LastArgType = F->getFunctionType()->getParamType(

87 F->getFunctionType()->getNumParams() - 1);

89 return false;

90

91

94 return true;

95}

96

97

98

101

102 if (F->getReturnType()->isVectorTy())

103 return false;

104

107 return true;

108}

109

110

111

114

115 Type *Arg1Type = F->getFunctionType()->getParamType(1);

116 Type *Arg2Type = F->getFunctionType()->getParamType(2);

118 cast(Arg1Type)->getElementType()->isIntegerTy(8) &&

120 cast(Arg2Type)->getElementType()->isIntegerTy(8))

121 return false;

122

125 return true;

126}

127

130 if (F->getReturnType()->getScalarType()->isBFloatTy())

131 return false;

132

135 return true;

136}

137

140 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())

141 return false;

142

145 return true;

146}

147

149

150

151

152

153

154 if (Name.consume_front("avx."))

155 return (Name.starts_with("blend.p") ||

156 Name == "cvt.ps2.pd.256" ||

157 Name == "cvtdq2.pd.256" ||

158 Name == "cvtdq2.ps.256" ||

159 Name.starts_with("movnt.") ||

160 Name.starts_with("sqrt.p") ||

161 Name.starts_with("storeu.") ||

162 Name.starts_with("vbroadcast.s") ||

163 Name.starts_with("vbroadcastf128") ||

164 Name.starts_with("vextractf128.") ||

165 Name.starts_with("vinsertf128.") ||

166 Name.starts_with("vperm2f128.") ||

167 Name.starts_with("vpermil."));

168

169 if (Name.consume_front("avx2."))

170 return (Name == "movntdqa" ||

171 Name.starts_with("pabs.") ||

172 Name.starts_with("padds.") ||

173 Name.starts_with("paddus.") ||

174 Name.starts_with("pblendd.") ||

175 Name == "pblendw" ||

176 Name.starts_with("pbroadcast") ||

177 Name.starts_with("pcmpeq.") ||

178 Name.starts_with("pcmpgt.") ||

179 Name.starts_with("pmax") ||

180 Name.starts_with("pmin") ||

181 Name.starts_with("pmovsx") ||

182 Name.starts_with("pmovzx") ||

183 Name == "pmul.dq" ||

184 Name == "pmulu.dq" ||

185 Name.starts_with("psll.dq") ||

186 Name.starts_with("psrl.dq") ||

187 Name.starts_with("psubs.") ||

188 Name.starts_with("psubus.") ||

189 Name.starts_with("vbroadcast") ||

190 Name == "vbroadcasti128" ||

191 Name == "vextracti128" ||

192 Name == "vinserti128" ||

193 Name == "vperm2i128");

194

195 if (Name.consume_front("avx512.")) {

196 if (Name.consume_front("mask."))

197

198 return (Name.starts_with("add.p") ||

199 Name.starts_with("and.") ||

200 Name.starts_with("andn.") ||

201 Name.starts_with("broadcast.s") ||

202 Name.starts_with("broadcastf32x4.") ||

203 Name.starts_with("broadcastf32x8.") ||

204 Name.starts_with("broadcastf64x2.") ||

205 Name.starts_with("broadcastf64x4.") ||

206 Name.starts_with("broadcasti32x4.") ||

207 Name.starts_with("broadcasti32x8.") ||

208 Name.starts_with("broadcasti64x2.") ||

209 Name.starts_with("broadcasti64x4.") ||

210 Name.starts_with("cmp.b") ||

211 Name.starts_with("cmp.d") ||

212 Name.starts_with("cmp.q") ||

213 Name.starts_with("cmp.w") ||

214 Name.starts_with("compress.b") ||

215 Name.starts_with("compress.d") ||

216 Name.starts_with("compress.p") ||

217 Name.starts_with("compress.q") ||

218 Name.starts_with("compress.store.") ||

219 Name.starts_with("compress.w") ||

220 Name.starts_with("conflict.") ||

221 Name.starts_with("cvtdq2pd.") ||

222 Name.starts_with("cvtdq2ps.") ||

223 Name == "cvtpd2dq.256" ||

224 Name == "cvtpd2ps.256" ||

225 Name == "cvtps2pd.128" ||

226 Name == "cvtps2pd.256" ||

227 Name.starts_with("cvtqq2pd.") ||

228 Name == "cvtqq2ps.256" ||

229 Name == "cvtqq2ps.512" ||

230 Name == "cvttpd2dq.256" ||

231 Name == "cvttps2dq.128" ||

232 Name == "cvttps2dq.256" ||

233 Name.starts_with("cvtudq2pd.") ||

234 Name.starts_with("cvtudq2ps.") ||

235 Name.starts_with("cvtuqq2pd.") ||

236 Name == "cvtuqq2ps.256" ||

237 Name == "cvtuqq2ps.512" ||

238 Name.starts_with("dbpsadbw.") ||

239 Name.starts_with("div.p") ||

240 Name.starts_with("expand.b") ||

241 Name.starts_with("expand.d") ||

242 Name.starts_with("expand.load.") ||

243 Name.starts_with("expand.p") ||

244 Name.starts_with("expand.q") ||

245 Name.starts_with("expand.w") ||

246 Name.starts_with("fpclass.p") ||

247 Name.starts_with("insert") ||

248 Name.starts_with("load.") ||

249 Name.starts_with("loadu.") ||

250 Name.starts_with("lzcnt.") ||

251 Name.starts_with("max.p") ||

252 Name.starts_with("min.p") ||

253 Name.starts_with("movddup") ||

254 Name.starts_with("move.s") ||

255 Name.starts_with("movshdup") ||

256 Name.starts_with("movsldup") ||

257 Name.starts_with("mul.p") ||

258 Name.starts_with("or.") ||

259 Name.starts_with("pabs.") ||

260 Name.starts_with("packssdw.") ||

261 Name.starts_with("packsswb.") ||

262 Name.starts_with("packusdw.") ||

263 Name.starts_with("packuswb.") ||

264 Name.starts_with("padd.") ||

265 Name.starts_with("padds.") ||

266 Name.starts_with("paddus.") ||

267 Name.starts_with("palignr.") ||

268 Name.starts_with("pand.") ||

269 Name.starts_with("pandn.") ||

270 Name.starts_with("pavg") ||

271 Name.starts_with("pbroadcast") ||

272 Name.starts_with("pcmpeq.") ||

273 Name.starts_with("pcmpgt.") ||

274 Name.starts_with("perm.df.") ||

275 Name.starts_with("perm.di.") ||

276 Name.starts_with("permvar.") ||

277 Name.starts_with("pmaddubs.w.") ||

278 Name.starts_with("pmaddw.d.") ||

279 Name.starts_with("pmax") ||

280 Name.starts_with("pmin") ||

281 Name == "pmov.qd.256" ||

282 Name == "pmov.qd.512" ||

283 Name == "pmov.wb.256" ||

284 Name == "pmov.wb.512" ||

285 Name.starts_with("pmovsx") ||

286 Name.starts_with("pmovzx") ||

287 Name.starts_with("pmul.dq.") ||

288 Name.starts_with("pmul.hr.sw.") ||

289 Name.starts_with("pmulh.w.") ||

290 Name.starts_with("pmulhu.w.") ||

291 Name.starts_with("pmull.") ||

292 Name.starts_with("pmultishift.qb.") ||

293 Name.starts_with("pmulu.dq.") ||

294 Name.starts_with("por.") ||

295 Name.starts_with("prol.") ||

296 Name.starts_with("prolv.") ||

297 Name.starts_with("pror.") ||

298 Name.starts_with("prorv.") ||

299 Name.starts_with("pshuf.b.") ||

300 Name.starts_with("pshuf.d.") ||

301 Name.starts_with("pshufh.w.") ||

302 Name.starts_with("pshufl.w.") ||

303 Name.starts_with("psll.d") ||

304 Name.starts_with("psll.q") ||

305 Name.starts_with("psll.w") ||

306 Name.starts_with("pslli") ||

307 Name.starts_with("psllv") ||

308 Name.starts_with("psra.d") ||

309 Name.starts_with("psra.q") ||

310 Name.starts_with("psra.w") ||

311 Name.starts_with("psrai") ||

312 Name.starts_with("psrav") ||

313 Name.starts_with("psrl.d") ||

314 Name.starts_with("psrl.q") ||

315 Name.starts_with("psrl.w") ||

316 Name.starts_with("psrli") ||

317 Name.starts_with("psrlv") ||

318 Name.starts_with("psub.") ||

319 Name.starts_with("psubs.") ||

320 Name.starts_with("psubus.") ||

321 Name.starts_with("pternlog.") ||

322 Name.starts_with("punpckh") ||

323 Name.starts_with("punpckl") ||

324 Name.starts_with("pxor.") ||

325 Name.starts_with("shuf.f") ||

326 Name.starts_with("shuf.i") ||

327 Name.starts_with("shuf.p") ||

328 Name.starts_with("sqrt.p") ||

329 Name.starts_with("store.b.") ||

330 Name.starts_with("store.d.") ||

331 Name.starts_with("store.p") ||

332 Name.starts_with("store.q.") ||

333 Name.starts_with("store.w.") ||

334 Name == "store.ss" ||

335 Name.starts_with("storeu.") ||

336 Name.starts_with("sub.p") ||

337 Name.starts_with("ucmp.") ||

338 Name.starts_with("unpckh.") ||

339 Name.starts_with("unpckl.") ||

340 Name.starts_with("valign.") ||

341 Name == "vcvtph2ps.128" ||

342 Name == "vcvtph2ps.256" ||

343 Name.starts_with("vextract") ||

344 Name.starts_with("vfmadd.") ||

345 Name.starts_with("vfmaddsub.") ||

346 Name.starts_with("vfnmadd.") ||

347 Name.starts_with("vfnmsub.") ||

348 Name.starts_with("vpdpbusd.") ||

349 Name.starts_with("vpdpbusds.") ||

350 Name.starts_with("vpdpwssd.") ||

351 Name.starts_with("vpdpwssds.") ||

352 Name.starts_with("vpermi2var.") ||

353 Name.starts_with("vpermil.p") ||

354 Name.starts_with("vpermilvar.") ||

355 Name.starts_with("vpermt2var.") ||

356 Name.starts_with("vpmadd52") ||

357 Name.starts_with("vpshld.") ||

358 Name.starts_with("vpshldv.") ||

359 Name.starts_with("vpshrd.") ||

360 Name.starts_with("vpshrdv.") ||

361 Name.starts_with("vpshufbitqmb.") ||

362 Name.starts_with("xor."));

363

364 if (Name.consume_front("mask3."))

365

366 return (Name.starts_with("vfmadd.") ||

367 Name.starts_with("vfmaddsub.") ||

368 Name.starts_with("vfmsub.") ||

369 Name.starts_with("vfmsubadd.") ||

370 Name.starts_with("vfnmsub."));

371

372 if (Name.consume_front("maskz."))

373

374 return (Name.starts_with("pternlog.") ||

375 Name.starts_with("vfmadd.") ||

376 Name.starts_with("vfmaddsub.") ||

377 Name.starts_with("vpdpbusd.") ||

378 Name.starts_with("vpdpbusds.") ||

379 Name.starts_with("vpdpwssd.") ||

380 Name.starts_with("vpdpwssds.") ||

381 Name.starts_with("vpermt2var.") ||

382 Name.starts_with("vpmadd52") ||

383 Name.starts_with("vpshldv.") ||

384 Name.starts_with("vpshrdv."));

385

386

387 return (Name == "movntdqa" ||

388 Name == "pmul.dq.512" ||

389 Name == "pmulu.dq.512" ||

390 Name.starts_with("broadcastm") ||

391 Name.starts_with("cmp.p") ||

392 Name.starts_with("cvtb2mask.") ||

393 Name.starts_with("cvtd2mask.") ||

394 Name.starts_with("cvtmask2") ||

395 Name.starts_with("cvtq2mask.") ||

396 Name == "cvtusi2sd" ||

397 Name.starts_with("cvtw2mask.") ||

398 Name == "kand.w" ||

399 Name == "kandn.w" ||

400 Name == "knot.w" ||

401 Name == "kor.w" ||

402 Name == "kortestc.w" ||

403 Name == "kortestz.w" ||

404 Name.starts_with("kunpck") ||

405 Name == "kxnor.w" ||

406 Name == "kxor.w" ||

407 Name.starts_with("padds.") ||

408 Name.starts_with("pbroadcast") ||

409 Name.starts_with("prol") ||

410 Name.starts_with("pror") ||

411 Name.starts_with("psll.dq") ||

412 Name.starts_with("psrl.dq") ||

413 Name.starts_with("psubs.") ||

414 Name.starts_with("ptestm") ||

415 Name.starts_with("ptestnm") ||

416 Name.starts_with("storent.") ||

417 Name.starts_with("vbroadcast.s") ||

418 Name.starts_with("vpshld.") ||

419 Name.starts_with("vpshrd."));

420 }

421

422 if (Name.consume_front("fma."))

423 return (Name.starts_with("vfmadd.") ||

424 Name.starts_with("vfmsub.") ||

425 Name.starts_with("vfmsubadd.") ||

426 Name.starts_with("vfnmadd.") ||

427 Name.starts_with("vfnmsub."));

428

429 if (Name.consume_front("fma4."))

430 return Name.starts_with("vfmadd.s");

431

432 if (Name.consume_front("sse."))

433 return (Name == "add.ss" ||

434 Name == "cvtsi2ss" ||

435 Name == "cvtsi642ss" ||

436 Name == "div.ss" ||

437 Name == "mul.ss" ||

438 Name.starts_with("sqrt.p") ||

439 Name == "sqrt.ss" ||

440 Name.starts_with("storeu.") ||

441 Name == "sub.ss");

442

443 if (Name.consume_front("sse2."))

444 return (Name == "add.sd" ||

445 Name == "cvtdq2pd" ||

446 Name == "cvtdq2ps" ||

447 Name == "cvtps2pd" ||

448 Name == "cvtsi2sd" ||

449 Name == "cvtsi642sd" ||

450 Name == "cvtss2sd" ||

451 Name == "div.sd" ||

452 Name == "mul.sd" ||

453 Name.starts_with("padds.") ||

454 Name.starts_with("paddus.") ||

455 Name.starts_with("pcmpeq.") ||

456 Name.starts_with("pcmpgt.") ||

457 Name == "pmaxs.w" ||

458 Name == "pmaxu.b" ||

459 Name == "pmins.w" ||

460 Name == "pminu.b" ||

461 Name == "pmulu.dq" ||

462 Name.starts_with("pshuf") ||

463 Name.starts_with("psll.dq") ||

464 Name.starts_with("psrl.dq") ||

465 Name.starts_with("psubs.") ||

466 Name.starts_with("psubus.") ||

467 Name.starts_with("sqrt.p") ||

468 Name == "sqrt.sd" ||

469 Name == "storel.dq" ||

470 Name.starts_with("storeu.") ||

471 Name == "sub.sd");

472

473 if (Name.consume_front("sse41."))

474 return (Name.starts_with("blendp") ||

475 Name == "movntdqa" ||

476 Name == "pblendw" ||

477 Name == "pmaxsb" ||

478 Name == "pmaxsd" ||

479 Name == "pmaxud" ||

480 Name == "pmaxuw" ||

481 Name == "pminsb" ||

482 Name == "pminsd" ||

483 Name == "pminud" ||

484 Name == "pminuw" ||

485 Name.starts_with("pmovsx") ||

486 Name.starts_with("pmovzx") ||

487 Name == "pmuldq");

488

489 if (Name.consume_front("sse42."))

490 return Name == "crc32.64.8";

491

492 if (Name.consume_front("sse4a."))

493 return Name.starts_with("movnt.");

494

495 if (Name.consume_front("ssse3."))

496 return (Name == "pabs.b.128" ||

497 Name == "pabs.d.128" ||

498 Name == "pabs.w.128");

499

500 if (Name.consume_front("xop."))

501 return (Name == "vpcmov" ||

502 Name == "vpcmov.256" ||

503 Name.starts_with("vpcom") ||

504 Name.starts_with("vprot"));

505

506 return (Name == "addcarry.u32" ||

507 Name == "addcarry.u64" ||

508 Name == "addcarryx.u32" ||

509 Name == "addcarryx.u64" ||

510 Name == "subborrow.u32" ||

511 Name == "subborrow.u64" ||

512 Name.starts_with("vcvtph2ps."));

513}

514

517

518 if (!Name.consume_front("x86."))

519 return false;

520

522 NewFn = nullptr;

523 return true;

524 }

525

526 if (Name == "rdtscp") {

527

528 if (F->getFunctionType()->getNumParams() == 0)

529 return false;

530

533 Intrinsic::x86_rdtscp);

534 return true;

535 }

536

538

539

540 if (Name.consume_front("sse41.ptest")) {

542 .Case("c", Intrinsic::x86_sse41_ptestc)

543 .Case("z", Intrinsic::x86_sse41_ptestz)

544 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)

548

549 return false;

550 }

551

552

553

554

555

557 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)

558 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)

559 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)

560 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)

561 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)

562 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)

566

567 if (Name.consume_front("avx512.")) {

568 if (Name.consume_front("mask.cmp.")) {

569

571 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)

572 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)

573 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)

574 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)

575 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)

576 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)

580 } else if (Name.starts_with("vpdpbusd.") ||

581 Name.starts_with("vpdpbusds.")) {

582

584 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)

585 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)

586 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)

587 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)

588 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)

589 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)

593 }

594 return false;

595 }

596

597 if (Name.consume_front("avx2.vpdpb")) {

598

600 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)

601 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)

602 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)

603 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)

604 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)

605 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)

606 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)

607 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)

608 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)

609 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)

610 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)

611 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)

615 return false;

616 }

617

618 if (Name.consume_front("avx10.vpdpb")) {

619

621 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)

622 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)

623 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)

624 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)

625 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)

626 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)

630 return false;

631 }

632

633 if (Name.consume_front("avx512bf16.")) {

634

636 .Case("cvtne2ps2bf16.128",

637 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)

638 .Case("cvtne2ps2bf16.256",

639 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)

640 .Case("cvtne2ps2bf16.512",

641 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)

642 .Case("mask.cvtneps2bf16.128",

643 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)

644 .Case("cvtneps2bf16.256",

645 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)

646 .Case("cvtneps2bf16.512",

647 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)

651

652

654 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)

655 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)

656 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)

660 return false;

661 }

662

663 if (Name.consume_front("xop.")) {

665 if (Name.starts_with("vpermil2")) {

666

667

668 auto Idx = F->getFunctionType()->getParamType(2);

669 if (Idx->isFPOrFPVectorTy()) {

670 unsigned IdxSize = Idx->getPrimitiveSizeInBits();

671 unsigned EltSize = Idx->getScalarSizeInBits();

672 if (EltSize == 64 && IdxSize == 128)

673 ID = Intrinsic::x86_xop_vpermil2pd;

674 else if (EltSize == 32 && IdxSize == 128)

675 ID = Intrinsic::x86_xop_vpermil2ps;

676 else if (EltSize == 64 && IdxSize == 256)

677 ID = Intrinsic::x86_xop_vpermil2pd_256;

678 else

679 ID = Intrinsic::x86_xop_vpermil2ps_256;

680 }

681 } else if (F->arg_size() == 2)

682

684 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)

685 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)

687

691 return true;

692 }

693 return false;

694 }

695

696 if (Name == "seh.recoverfp") {

698 Intrinsic::eh_recoverfp);

699 return true;

700 }

701

702 return false;

703}

704

705

706

710 if (Name.starts_with("rbit")) {

711

713 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());

714 return true;

715 }

716

717 if (Name == "thread.pointer") {

718

720 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());

721 return true;

722 }

723

724 bool Neon = Name.consume_front("neon.");

725 if (Neon) {

726

727

728

729 if (Name.consume_front("bfdot.")) {

730

733 .Cases({"v2f32.v8i8", "v4f32.v16i8"},

738 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();

739 assert((OperandWidth == 64 || OperandWidth == 128) &&

740 "Unexpected operand width");

741 LLVMContext &Ctx = F->getParent()->getContext();

742 std::array<Type *, 2> Tys{

743 {F->getReturnType(),

746 return true;

747 }

748 return false;

749 }

750

751

752

753 if (Name.consume_front("bfm")) {

754

755 if (Name.consume_back(".v4f32.v16i8")) {

756

760 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla

762 .Case("lalb",

763 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb

764 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)

765 .Case("lalt",

766 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt

767 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)

771 return true;

772 }

773 return false;

774 }

775 return false;

776 }

777

778 }

779

780

781 if (IsArm) {

782

783 if (Neon) {

784

786 .StartsWith("vclz.", Intrinsic::ctlz)

787 .StartsWith("vcnt.", Intrinsic::ctpop)

788 .StartsWith("vqadds.", Intrinsic::sadd_sat)

789 .StartsWith("vqaddu.", Intrinsic::uadd_sat)

790 .StartsWith("vqsubs.", Intrinsic::ssub_sat)

791 .StartsWith("vqsubu.", Intrinsic::usub_sat)

792 .StartsWith("vrinta.", Intrinsic::round)

793 .StartsWith("vrintn.", Intrinsic::roundeven)

794 .StartsWith("vrintm.", Intrinsic::floor)

795 .StartsWith("vrintp.", Intrinsic::ceil)

796 .StartsWith("vrintx.", Intrinsic::rint)

797 .StartsWith("vrintz.", Intrinsic::trunc)

801 F->arg_begin()->getType());

802 return true;

803 }

804

805 if (Name.consume_front("vst")) {

806

807 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");

811 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,

812 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};

813

815 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,

816 Intrinsic::arm_neon_vst4lane};

817

818 auto fArgs = F->getFunctionType()->params();

819 Type *Tys[] = {fArgs[0], fArgs[1]};

822 F->getParent(), StoreInts[fArgs.size() - 3], Tys);

823 else

825 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);

826 return true;

827 }

828 return false;

829 }

830

831 return false;

832 }

833

834 if (Name.consume_front("mve.")) {

835

836 if (Name == "vctp64") {

838

839

841 return true;

842 }

843 return false;

844 }

845

846 if (Name.starts_with("vrintn.v")) {

848 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());

849 return true;

850 }

851

852

853 if (Name.consume_back(".v4i1")) {

854

855 if (Name.consume_back(".predicated.v2i64.v4i32"))

856

857 return Name == "mull.int" || Name == "vqdmull";

858

859 if (Name.consume_back(".v2i64")) {

860

861 bool IsGather = Name.consume_front("vldr.gather.");

862 if (IsGather || Name.consume_front("vstr.scatter.")) {

863 if (Name.consume_front("base.")) {

864

865 Name.consume_front("wb.");

866

867

868 return Name == "predicated.v2i64";

869 }

870

871 if (Name.consume_front("offset.predicated."))

872 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||

873 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");

874

875

876 return false;

877 }

878

879 return false;

880 }

881 return false;

882 }

883 return false;

884 }

885

886 if (Name.consume_front("cde.vcx")) {

887

888 if (Name.consume_back(".predicated.v2i64.v4i1"))

889

890 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||

891 Name == "3q" || Name == "3qa";

892

893 return false;

894 }

895 } else {

896

897 if (Neon) {

898

900 .StartsWith("frintn", Intrinsic::roundeven)

901 .StartsWith("rbit", Intrinsic::bitreverse)

905 F->arg_begin()->getType());

906 return true;

907 }

908

909 if (Name.starts_with("addp")) {

910

911 if (F->arg_size() != 2)

912 return false;

914 if (Ty && Ty->getElementType()->isFloatingPointTy()) {

916 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);

917 return true;

918 }

919 }

920

921

922 if (Name.starts_with("bfcvt")) {

923 NewFn = nullptr;

924 return true;

925 }

926

927 return false;

928 }

929 if (Name.consume_front("sve.")) {

930

931 if (Name.consume_front("bf")) {

932 if (Name.consume_back(".lane")) {

933

936 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)

937 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)

938 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)

942 return true;

943 }

944 return false;

945 }

946 return false;

947 }

948

949

950 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {

951 NewFn = nullptr;

952 return true;

953 }

954

955 if (Name.consume_front("addqv")) {

956

957 if (->getReturnType()->isFPOrFPVectorTy())

958 return false;

959

960 auto Args = F->getFunctionType()->params();

961 Type *Tys[] = {F->getReturnType(), Args[1]};

963 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);

964 return true;

965 }

966

967 if (Name.consume_front("ld")) {

968

969 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");

970 if (LdRegex.match(Name)) {

971 Type *ScalarTy =

977 Intrinsic::aarch64_sve_ld2_sret,

978 Intrinsic::aarch64_sve_ld3_sret,

979 Intrinsic::aarch64_sve_ld4_sret,

980 };

982 LoadIDs[Name[0] - '2'], Ty);

983 return true;

984 }

985 return false;

986 }

987

988 if (Name.consume_front("tuple.")) {

989

990 if (Name.starts_with("get")) {

991

992 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};

994 F->getParent(), Intrinsic::vector_extract, Tys);

995 return true;

996 }

997

998 if (Name.starts_with("set")) {

999

1000 auto Args = F->getFunctionType()->params();

1001 Type *Tys[] = {Args[0], Args[2], Args[1]};

1003 F->getParent(), Intrinsic::vector_insert, Tys);

1004 return true;

1005 }

1006

1007 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");

1008 if (CreateTupleRegex.match(Name)) {

1009

1010 auto Args = F->getFunctionType()->params();

1011 Type *Tys[] = {F->getReturnType(), Args[1]};

1013 F->getParent(), Intrinsic::vector_insert, Tys);

1014 return true;

1015 }

1016 return false;

1017 }

1018

1019 if (Name.starts_with("rev.nxv")) {

1020

1022 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());

1023 return true;

1024 }

1025

1026 return false;

1027 }

1028 }

1029 return false;

1030}

1031

1034 if (Name.consume_front("cp.async.bulk.tensor.g2s.")) {

1037 .Case("im2col.3d",

1038 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d)

1039 .Case("im2col.4d",

1040 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d)

1041 .Case("im2col.5d",

1042 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d)

1043 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d)

1044 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d)

1045 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d)

1046 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d)

1047 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d)

1049

1051 return ID;

1052

1053

1054

1055

1056 if (F->getArg(0)->getType()->getPointerAddressSpace() ==

1058 return ID;

1059

1060

1061

1062

1063

1064

1065

1066

1067

1068

1069

1070 size_t FlagStartIndex = F->getFunctionType()->getNumParams() - 3;

1071 Type *ArgType = F->getFunctionType()->getParamType(FlagStartIndex);

1073 return ID;

1074 }

1075

1077}

1078

1081 if (Name.consume_front("mapa.shared.cluster"))

1082 if (F->getReturnType()->getPointerAddressSpace() ==

1084 return Intrinsic::nvvm_mapa_shared_cluster;

1085

1086 if (Name.consume_front("cp.async.bulk.")) {

1089 .Case("global.to.shared.cluster",

1090 Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)

1091 .Case("shared.cta.to.cluster",

1092 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)

1094

1096 if (F->getArg(0)->getType()->getPointerAddressSpace() ==

1098 return ID;

1099 }

1100

1102}

1103

1105 if (Name.consume_front("fma.rn."))

1107 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)

1108 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)

1109 .Case("ftz.bf16", Intrinsic::nvvm_fma_rn_ftz_bf16)

1110 .Case("ftz.bf16x2", Intrinsic::nvvm_fma_rn_ftz_bf16x2)

1111 .Case("ftz.relu.bf16", Intrinsic::nvvm_fma_rn_ftz_relu_bf16)

1112 .Case("ftz.relu.bf16x2", Intrinsic::nvvm_fma_rn_ftz_relu_bf16x2)

1113 .Case("ftz.sat.bf16", Intrinsic::nvvm_fma_rn_ftz_sat_bf16)

1114 .Case("ftz.sat.bf16x2", Intrinsic::nvvm_fma_rn_ftz_sat_bf16x2)

1115 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)

1116 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)

1117 .Case("sat.bf16", Intrinsic::nvvm_fma_rn_sat_bf16)

1118 .Case("sat.bf16x2", Intrinsic::nvvm_fma_rn_sat_bf16x2)

1120

1121 if (Name.consume_front("fmax."))

1123 .Case("bf16", Intrinsic::nvvm_fmax_bf16)

1124 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)

1125 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)

1126 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)

1127 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)

1128 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)

1129 .Case("ftz.nan.xorsign.abs.bf16",

1130 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)

1131 .Case("ftz.nan.xorsign.abs.bf16x2",

1132 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)

1133 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)

1134 .Case("ftz.xorsign.abs.bf16x2",

1135 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)

1136 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)

1137 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)

1138 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)

1139 .Case("nan.xorsign.abs.bf16x2",

1140 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)

1141 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)

1142 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)

1144

1145 if (Name.consume_front("fmin."))

1147 .Case("bf16", Intrinsic::nvvm_fmin_bf16)

1148 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)

1149 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)

1150 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)

1151 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)

1152 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)

1153 .Case("ftz.nan.xorsign.abs.bf16",

1154 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)

1155 .Case("ftz.nan.xorsign.abs.bf16x2",

1156 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)

1157 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)

1158 .Case("ftz.xorsign.abs.bf16x2",

1159 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)

1160 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)

1161 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)

1162 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)

1163 .Case("nan.xorsign.abs.bf16x2",

1164 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)

1165 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)

1166 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)

1168

1169 if (Name.consume_front("neg."))

1171 .Case("bf16", Intrinsic::nvvm_neg_bf16)

1172 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)

1174

1176}

1177

1179 return Name.consume_front("local") || Name.consume_front("shared") ||

1180 Name.consume_front("global") || Name.consume_front("constant") ||

1181 Name.consume_front("param");

1182}

1183

1185 bool CanUpgradeDebugIntrinsicsToRecords) {

1186 assert(F && "Illegal to upgrade a non-existent Function.");

1187

1189

1190

1191 if (!Name.consume_front("llvm.") || Name.empty())

1192 return false;

1193

1194 switch (Name[0]) {

1195 default: break;

1196 case 'a': {

1197 bool IsArm = Name.consume_front("arm.");

1198 if (IsArm || Name.consume_front("aarch64.")) {

1200 return true;

1201 break;

1202 }

1203

1204 if (Name.consume_front("amdgcn.")) {

1205 if (Name == "alignbit") {

1206

1208 F->getParent(), Intrinsic::fshr, {F->getReturnType()});

1209 return true;

1210 }

1211

1212 if (Name.consume_front("atomic.")) {

1213 if (Name.starts_with("inc") || Name.starts_with("dec")) {

1214

1215

1216 NewFn = nullptr;

1217 return true;

1218 }

1219 break;

1220 }

1221

1222 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||

1223 Name.consume_front("flat.atomic.")) {

1224 if (Name.starts_with("fadd") ||

1225

1226 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||

1227 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {

1228

1229

1230 NewFn = nullptr;

1231 return true;

1232 }

1233 }

1234

1235 if (Name.starts_with("ldexp.")) {

1236

1238 F->getParent(), Intrinsic::ldexp,

1239 {F->getReturnType(), F->getArg(1)->getType()});

1240 return true;

1241 }

1242 break;

1243 }

1244

1245 break;

1246 }

1247 case 'c': {

1248 if (F->arg_size() == 1) {

1250 .StartsWith("ctlz.", Intrinsic::ctlz)

1251 .StartsWith("cttz.", Intrinsic::cttz)

1256 F->arg_begin()->getType());

1257 return true;

1258 }

1259 }

1260

1261 if (F->arg_size() == 2 && Name == "coro.end") {

1264 Intrinsic::coro_end);

1265 return true;

1266 }

1267

1268 break;

1269 }

1270 case 'd':

1271 if (Name.consume_front("dbg.")) {

1272

1273 if (CanUpgradeDebugIntrinsicsToRecords) {

1274 if (Name == "addr" || Name == "value" || Name == "assign" ||

1275 Name == "declare" || Name == "label") {

1276

1277 NewFn = nullptr;

1278

1279 return true;

1280 }

1281 }

1282

1283

1284 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {

1287 Intrinsic::dbg_value);

1288 return true;

1289 }

1290 break;

1291 }

1292 break;

1293 case 'e':

1294 if (Name.consume_front("experimental.vector.")) {

1297

1298

1300 .StartsWith("extract.", Intrinsic::vector_extract)

1301 .StartsWith("insert.", Intrinsic::vector_insert)

1302 .StartsWith("splice.", Intrinsic::vector_splice)

1303 .StartsWith("reverse.", Intrinsic::vector_reverse)

1304 .StartsWith("interleave2.", Intrinsic::vector_interleave2)

1305 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)

1307 Intrinsic::vector_partial_reduce_add)

1310 const auto *FT = F->getFunctionType();

1312 if (ID == Intrinsic::vector_extract ||

1313 ID == Intrinsic::vector_interleave2)

1314

1315 Tys.push_back(FT->getReturnType());

1316 if (ID != Intrinsic::vector_interleave2)

1317 Tys.push_back(FT->getParamType(0));

1318 if (ID == Intrinsic::vector_insert ||

1319 ID == Intrinsic::vector_partial_reduce_add)

1320

1321 Tys.push_back(FT->getParamType(1));

1324 return true;

1325 }

1326

1327 if (Name.consume_front("reduce.")) {

1329 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");

1330 if (R.match(Name, &Groups))

1332 .Case("add", Intrinsic::vector_reduce_add)

1333 .Case("mul", Intrinsic::vector_reduce_mul)

1334 .Case("and", Intrinsic::vector_reduce_and)

1335 .Case("or", Intrinsic::vector_reduce_or)

1336 .Case("xor", Intrinsic::vector_reduce_xor)

1337 .Case("smax", Intrinsic::vector_reduce_smax)

1338 .Case("smin", Intrinsic::vector_reduce_smin)

1339 .Case("umax", Intrinsic::vector_reduce_umax)

1340 .Case("umin", Intrinsic::vector_reduce_umin)

1341 .Case("fmax", Intrinsic::vector_reduce_fmax)

1342 .Case("fmin", Intrinsic::vector_reduce_fmin)

1344

1345 bool V2 = false;

1347 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");

1349 V2 = true;

1350 if (R2.match(Name, &Groups))

1352 .Case("fadd", Intrinsic::vector_reduce_fadd)

1353 .Case("fmul", Intrinsic::vector_reduce_fmul)

1355 }

1358 auto Args = F->getFunctionType()->params();

1360 {Args[V2 ? 1 : 0]});

1361 return true;

1362 }

1363 break;

1364 }

1365 break;

1366 }

1367 if (Name.consume_front("experimental.stepvector.")) {

1371 F->getParent(), ID, F->getFunctionType()->getReturnType());

1372 return true;

1373 }

1374 break;

1375 case 'f':

1376 if (Name.starts_with("flt.rounds")) {

1379 Intrinsic::get_rounding);

1380 return true;

1381 }

1382 break;

1383 case 'i':

1384 if (Name.starts_with("invariant.group.barrier")) {

1385

1386 auto Args = F->getFunctionType()->params();

1387 Type* ObjectPtr[1] = {Args[0]};

1390 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);

1391 return true;

1392 }

1393 break;

1394 case 'l':

1395 if ((Name.starts_with("lifetime.start") ||

1396 Name.starts_with("lifetime.end")) &&

1397 F->arg_size() == 2) {

1398 Intrinsic::ID IID = Name.starts_with("lifetime.start")

1399 ? Intrinsic::lifetime_start

1400 : Intrinsic::lifetime_end;

1403 F->getArg(0)->getType());

1404 return true;

1405 }

1406 break;

1407 case 'm': {

1408

1409

1410

1412 .StartsWith("memcpy.", Intrinsic::memcpy)

1413 .StartsWith("memmove.", Intrinsic::memmove)

1415 if (F->arg_size() == 5) {

1417

1419 F->getFunctionType()->params().slice(0, 3);

1420 NewFn =

1422 return true;

1423 }

1424 }

1425 if (Name.starts_with("memset.") && F->arg_size() == 5) {

1427

1428 const auto *FT = F->getFunctionType();

1429 Type *ParamTypes[2] = {

1430 FT->getParamType(0),

1431 FT->getParamType(2)

1432 };

1434 Intrinsic::memset, ParamTypes);

1435 return true;

1436 }

1437

1438 unsigned MaskedID =

1440 .StartsWith("masked.load", Intrinsic::masked_load)

1441 .StartsWith("masked.gather", Intrinsic::masked_gather)

1442 .StartsWith("masked.store", Intrinsic::masked_store)

1443 .StartsWith("masked.scatter", Intrinsic::masked_scatter)

1445 if (MaskedID && F->arg_size() == 4) {

1447 if (MaskedID == Intrinsic::masked_load ||

1448 MaskedID == Intrinsic::masked_gather) {

1450 F->getParent(), MaskedID,

1451 {F->getReturnType(), F->getArg(0)->getType()});

1452 return true;

1453 }

1455 F->getParent(), MaskedID,

1456 {F->getArg(0)->getType(), F->getArg(1)->getType()});

1457 return true;

1458 }

1459 break;

1460 }

1461 case 'n': {

1462 if (Name.consume_front("nvvm.")) {

1463

1464 if (F->arg_size() == 1) {

1467 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)

1468 .Case("clz.i", Intrinsic::ctlz)

1469 .Case("popc.i", Intrinsic::ctpop)

1473 {F->getReturnType()});

1474 return true;

1475 }

1476 }

1477

1478

1479 if (->getReturnType()->getScalarType()->isBFloatTy()) {

1482 NewFn = nullptr;

1483 return true;

1484 }

1485 }

1486

1487

1492 return true;

1493 }

1494

1495

1500 return true;

1501 }

1502

1503

1504

1505

1506

1507 bool Expand = false;

1508 if (Name.consume_front("abs."))

1509

1510 Expand =

1511 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";

1512 else if (Name.consume_front("fabs."))

1513

1514 Expand = Name == "f" || Name == "ftz.f" || Name == "d";

1515 else if (Name.consume_front("ex2.approx."))

1516

1517 Expand =

1518 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";

1519 else if (Name.consume_front("max.") || Name.consume_front("min."))

1520

1521 Expand = Name == "s" || Name == "i" || Name == "ll" || Name == "us" ||

1522 Name == "ui" || Name == "ull";

1523 else if (Name.consume_front("atomic.load."))

1524

1525

1532 else if (Name.consume_front("bitcast."))

1533

1534 Expand =

1535 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";

1536 else if (Name.consume_front("rotate."))

1537

1538 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";

1539 else if (Name.consume_front("ptr.gen.to."))

1540

1542 else if (Name.consume_front("ptr."))

1543

1545 else if (Name.consume_front("ldg.global."))

1546

1547 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||

1548 Name.starts_with("p."));

1549 else

1551 .Case("barrier0", true)

1552 .Case("barrier.n", true)

1553 .Case("barrier.sync.cnt", true)

1554 .Case("barrier.sync", true)

1555 .Case("barrier", true)

1556 .Case("bar.sync", true)

1557 .Case("clz.ll", true)

1558 .Case("popc.ll", true)

1559 .Case("h2f", true)

1560 .Case("swap.lo.hi.b64", true)

1561 .Case("tanh.approx.f32", true)

1563

1564 if (Expand) {

1565 NewFn = nullptr;

1566 return true;

1567 }

1568 break;

1569 }

1570 break;

1571 }

1572 case 'o':

1573 if (Name.starts_with("objectsize.")) {

1574 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };

1575 if (F->arg_size() == 2 || F->arg_size() == 3) {

1578 Intrinsic::objectsize, Tys);

1579 return true;

1580 }

1581 }

1582 break;

1583

1584 case 'p':

1585 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {

1588 F->getParent(), Intrinsic::ptr_annotation,

1589 {F->arg_begin()->getType(), F->getArg(1)->getType()});

1590 return true;

1591 }

1592 break;

1593

1594 case 'r': {

1595 if (Name.consume_front("riscv.")) {

1598 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)

1599 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)

1600 .Case("aes32esi", Intrinsic::riscv_aes32esi)

1601 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)

1604 if (->getFunctionType()->getParamType(2)->isIntegerTy(32)) {

1607 return true;

1608 }

1609 break;

1610 }

1611

1613 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)

1614 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)

1617 if (->getFunctionType()->getParamType(2)->isIntegerTy(32) ||

1618 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {

1621 return true;

1622 }

1623 break;

1624 }

1625

1627 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)

1628 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)

1629 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)

1630 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)

1631 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)

1632 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)

1635 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {

1638 return true;

1639 }

1640 break;

1641 }

1642 break;

1643 }

1644 } break;

1645

1646 case 's':

1647 if (Name == "stackprotectorcheck") {

1648 NewFn = nullptr;

1649 return true;

1650 }

1651 break;

1652

1653 case 't':

1654 if (Name == "thread.pointer") {

1656 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());

1657 return true;

1658 }

1659 break;

1660

1661 case 'v': {

1662 if (Name == "var.annotation" && F->arg_size() == 4) {

1665 F->getParent(), Intrinsic::var_annotation,

1666 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});

1667 return true;

1668 }

1669 break;

1670 }

1671

1672 case 'w':

1673 if (Name.consume_front("wasm.")) {

1676 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)

1677 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)

1678 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)

1683 F->getReturnType());

1684 return true;

1685 }

1686

1687 if (Name.consume_front("dot.i8x16.i7x16.")) {

1689 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)

1690 .Case("add.signed",

1691 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)

1696 return true;

1697 }

1698 break;

1699 }

1700 break;

1701 }

1702 break;

1703

1704 case 'x':

1706 return true;

1707 }

1708

1710 if (ST && (->isLiteral() || ST->isPacked()) &&

1712

1713

1714

1715

1719 auto *FT = F->getFunctionType();

1721 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());

1722 std::string Name = F->getName().str();

1724 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),

1725 Name, F->getParent());

1726

1727

1730 return true;

1731 }

1732 }

1733

1734

1736 if (Result != std::nullopt) {

1738 return true;

1739 }

1740

1741

1742

1743

1744

1745 return false;

1746}

1747

1749 bool CanUpgradeDebugIntrinsicsToRecords) {

1750 NewFn = nullptr;

1751 bool Upgraded =

1753

1754

1755 if (NewFn)

1756 F = NewFn;

1758

1761 F->setAttributes(

1763 }

1764 return Upgraded;

1765}

1766

1768 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||

1769 GV->getName() == "llvm.global_dtors")) ||

1771 return nullptr;

1773 if (!ATy)

1774 return nullptr;

1777 return nullptr;

1778

1784 unsigned N = Init->getNumOperands();

1785 std::vector<Constant *> NewCtors(N);

1786 for (unsigned i = 0; i != N; ++i) {

1789 Ctor->getAggregateElement(1),

1791 }

1793

1795 NewInit, GV->getName());

1796}

1797

1798

1799

1801 unsigned Shift) {

1803 unsigned NumElts = ResultTy->getNumElements() * 8;

1804

1805

1807 Op = Builder.CreateBitCast(Op, VecTy, "cast");

1808

1809

1811

1812

1813

1814 if (Shift < 16) {

1815 int Idxs[64];

1816

1817 for (unsigned l = 0; l != NumElts; l += 16)

1818 for (unsigned i = 0; i != 16; ++i) {

1819 unsigned Idx = NumElts + i - Shift;

1820 if (Idx < NumElts)

1821 Idx -= NumElts - 16;

1822 Idxs[l + i] = Idx + l;

1823 }

1824

1825 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));

1826 }

1827

1828

1829 return Builder.CreateBitCast(Res, ResultTy, "cast");

1830}

1831

1832

1833

1835 unsigned Shift) {

1837 unsigned NumElts = ResultTy->getNumElements() * 8;

1838

1839

1841 Op = Builder.CreateBitCast(Op, VecTy, "cast");

1842

1843

1845

1846

1847

1848 if (Shift < 16) {

1849 int Idxs[64];

1850

1851 for (unsigned l = 0; l != NumElts; l += 16)

1852 for (unsigned i = 0; i != 16; ++i) {

1853 unsigned Idx = i + Shift;

1854 if (Idx >= 16)

1855 Idx += NumElts - 16;

1856 Idxs[l + i] = Idx + l;

1857 }

1858

1859 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));

1860 }

1861

1862

1863 return Builder.CreateBitCast(Res, ResultTy, "cast");

1864}

1865

1867 unsigned NumElts) {

1870 Builder.getInt1Ty(), cast(Mask->getType())->getBitWidth());

1871 Mask = Builder.CreateBitCast(Mask, MaskTy);

1872

1873

1874

1875 if (NumElts <= 4) {

1876 int Indices[4];

1877 for (unsigned i = 0; i != NumElts; ++i)

1878 Indices[i] = i;

1879 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),

1880 "extract");

1881 }

1882

1883 return Mask;

1884}

1885

1888

1890 if (C->isAllOnesValue())

1891 return Op0;

1892

1895 return Builder.CreateSelect(Mask, Op0, Op1);

1896}

1897

1900

1902 if (C->isAllOnesValue())

1903 return Op0;

1904

1906 Mask->getType()->getIntegerBitWidth());

1907 Mask = Builder.CreateBitCast(Mask, MaskTy);

1908 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);

1909 return Builder.CreateSelect(Mask, Op0, Op1);

1910}

1911

1912

1913

1914

1918 bool IsVALIGN) {

1920

1922 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");

1923 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");

1925

1926

1927 if (IsVALIGN)

1928 ShiftVal &= (NumElts - 1);

1929

1930

1931

1932 if (ShiftVal >= 32)

1934

1935

1936

1937 if (ShiftVal > 16) {

1938 ShiftVal -= 16;

1939 Op1 = Op0;

1941 }

1942

1943 int Indices[64];

1944

1945 for (unsigned l = 0; l < NumElts; l += 16) {

1946 for (unsigned i = 0; i != 16; ++i) {

1947 unsigned Idx = ShiftVal + i;

1948 if (!IsVALIGN && Idx >= 16)

1949 Idx += NumElts - 16;

1950 Indices[l + i] = Idx + l;

1951 }

1952 }

1953

1954 Value *Align = Builder.CreateShuffleVector(

1955 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");

1956

1958}

1959

1961 bool ZeroMask, bool IndexForm) {

1964 unsigned EltWidth = Ty->getScalarSizeInBits();

1965 bool IsFloat = Ty->isFPOrFPVectorTy();

1967 if (VecWidth == 128 && EltWidth == 32 && IsFloat)

1968 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;

1969 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)

1970 IID = Intrinsic::x86_avx512_vpermi2var_d_128;

1971 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)

1972 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;

1973 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)

1974 IID = Intrinsic::x86_avx512_vpermi2var_q_128;

1975 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)

1976 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;

1977 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)

1978 IID = Intrinsic::x86_avx512_vpermi2var_d_256;

1979 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)

1980 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;

1981 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)

1982 IID = Intrinsic::x86_avx512_vpermi2var_q_256;

1983 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)

1984 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;

1985 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)

1986 IID = Intrinsic::x86_avx512_vpermi2var_d_512;

1987 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)

1988 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;

1989 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)

1990 IID = Intrinsic::x86_avx512_vpermi2var_q_512;

1991 else if (VecWidth == 128 && EltWidth == 16)

1992 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;

1993 else if (VecWidth == 256 && EltWidth == 16)

1994 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;

1995 else if (VecWidth == 512 && EltWidth == 16)

1996 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;

1997 else if (VecWidth == 128 && EltWidth == 8)

1998 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;

1999 else if (VecWidth == 256 && EltWidth == 8)

2000 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;

2001 else if (VecWidth == 512 && EltWidth == 8)

2002 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;

2003 else

2005

2008

2009

2010 if (!IndexForm)

2012

2013 Value *V = Builder.CreateIntrinsic(IID, Args);

2016 Ty);

2018}

2019

2025 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});

2026

2027 if (CI.arg_size() == 4) {

2030 Res = emitX86Select(Builder, Mask, Res, VecSrc);

2031 }

2032 return Res;

2033}

2034

2036 bool IsRotateRight) {

2040

2041

2042

2043

2044 if (Amt->getType() != Ty) {

2046 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);

2047 Amt = Builder.CreateVectorSplat(NumElts, Amt);

2048 }

2049

2050 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;

2051 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});

2052

2053 if (CI.arg_size() == 4) {

2056 Res = emitX86Select(Builder, Mask, Res, VecSrc);

2057 }

2058 return Res;

2059}

2060

2062 bool IsSigned) {

2066

2068 switch (Imm) {

2069 case 0x0:

2071 break;

2072 case 0x1:

2074 break;

2075 case 0x2:

2077 break;

2078 case 0x3:

2080 break;

2081 case 0x4:

2083 break;

2084 case 0x5:

2086 break;

2087 case 0x6:

2089 case 0x7:

2091 default:

2093 }

2094

2095 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);

2096 Value *Ext = Builder.CreateSExt(Cmp, Ty);

2097 return Ext;

2098}

2099

2101 bool IsShiftRight, bool ZeroMask) {

2106

2107 if (IsShiftRight)

2109

2110

2111

2112

2113 if (Amt->getType() != Ty) {

2115 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);

2116 Amt = Builder.CreateVectorSplat(NumElts, Amt);

2117 }

2118

2119 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;

2120 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});

2121

2122 unsigned NumArgs = CI.arg_size();

2123 if (NumArgs >= 4) {

2128 Res = emitX86Select(Builder, Mask, Res, VecSrc);

2129 }

2130 return Res;

2131}

2132

2135 const Align Alignment =

2137 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)

2139

2140

2142 if (C->isAllOnesValue())

2143 return Builder.CreateAlignedStore(Data, Ptr, Alignment);

2144

2145

2148 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);

2149}

2150

2154 const Align Alignment =

2158 8)

2160

2161

2163 if (C->isAllOnesValue())

2164 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);

2165

2166

2169 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);

2170}

2171

2175 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,

2176 {Op0, Builder.getInt1(false)});

2179 return Res;

2180}

2181

2184

2185

2188

2189 if (IsSigned) {

2190

2191 Constant *ShiftAmt = ConstantInt::get(Ty, 32);

2192 LHS = Builder.CreateShl(LHS, ShiftAmt);

2193 LHS = Builder.CreateAShr(LHS, ShiftAmt);

2194 RHS = Builder.CreateShl(RHS, ShiftAmt);

2195 RHS = Builder.CreateAShr(RHS, ShiftAmt);

2196 } else {

2197

2198 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);

2199 LHS = Builder.CreateAnd(LHS, Mask);

2200 RHS = Builder.CreateAnd(RHS, Mask);

2201 }

2202

2203 Value *Res = Builder.CreateMul(LHS, RHS);

2204

2207

2208 return Res;

2209}

2210

2211

2215 if (Mask) {

2217 if ( || ->isAllOnesValue())

2218 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));

2219 }

2220

2221 if (NumElts < 8) {

2222 int Indices[8];

2223 for (unsigned i = 0; i != NumElts; ++i)

2224 Indices[i] = i;

2225 for (unsigned i = NumElts; i != 8; ++i)

2226 Indices[i] = NumElts + i % NumElts;

2227 Vec = Builder.CreateShuffleVector(Vec,

2229 Indices);

2230 }

2231 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));

2232}

2233

2235 unsigned CC, bool Signed) {

2238

2240 if (CC == 3) {

2243 } else if (CC == 7) {

2246 } else {

2248 switch (CC) {

2256 }

2257 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));

2258 }

2259

2261

2263}

2264

2265

2272

2278

2279 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));

2280 Value* Cmp = Builder.CreateIsNotNull(AndNode);

2281 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);

2282 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);

2283 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);

2284 return Builder.CreateInsertElement(A, Select, (uint64_t)0);

2285}

2286

2292 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");

2293}

2294

2295

2298 Name = Name.substr(12);

2299

2303 if (Name.starts_with("max.p")) {

2304 if (VecWidth == 128 && EltWidth == 32)

2305 IID = Intrinsic::x86_sse_max_ps;

2306 else if (VecWidth == 128 && EltWidth == 64)

2307 IID = Intrinsic::x86_sse2_max_pd;

2308 else if (VecWidth == 256 && EltWidth == 32)

2309 IID = Intrinsic::x86_avx_max_ps_256;

2310 else if (VecWidth == 256 && EltWidth == 64)

2311 IID = Intrinsic::x86_avx_max_pd_256;

2312 else

2314 } else if (Name.starts_with("min.p")) {

2315 if (VecWidth == 128 && EltWidth == 32)

2316 IID = Intrinsic::x86_sse_min_ps;

2317 else if (VecWidth == 128 && EltWidth == 64)

2318 IID = Intrinsic::x86_sse2_min_pd;

2319 else if (VecWidth == 256 && EltWidth == 32)

2320 IID = Intrinsic::x86_avx_min_ps_256;

2321 else if (VecWidth == 256 && EltWidth == 64)

2322 IID = Intrinsic::x86_avx_min_pd_256;

2323 else

2325 } else if (Name.starts_with("pshuf.b.")) {

2326 if (VecWidth == 128)

2327 IID = Intrinsic::x86_ssse3_pshuf_b_128;

2328 else if (VecWidth == 256)

2329 IID = Intrinsic::x86_avx2_pshuf_b;

2330 else if (VecWidth == 512)

2331 IID = Intrinsic::x86_avx512_pshuf_b_512;

2332 else

2334 } else if (Name.starts_with("pmul.hr.sw.")) {

2335 if (VecWidth == 128)

2336 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;

2337 else if (VecWidth == 256)

2338 IID = Intrinsic::x86_avx2_pmul_hr_sw;

2339 else if (VecWidth == 512)

2340 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;

2341 else

2343 } else if (Name.starts_with("pmulh.w.")) {

2344 if (VecWidth == 128)

2345 IID = Intrinsic::x86_sse2_pmulh_w;

2346 else if (VecWidth == 256)

2347 IID = Intrinsic::x86_avx2_pmulh_w;

2348 else if (VecWidth == 512)

2349 IID = Intrinsic::x86_avx512_pmulh_w_512;

2350 else

2352 } else if (Name.starts_with("pmulhu.w.")) {

2353 if (VecWidth == 128)

2354 IID = Intrinsic::x86_sse2_pmulhu_w;

2355 else if (VecWidth == 256)

2356 IID = Intrinsic::x86_avx2_pmulhu_w;

2357 else if (VecWidth == 512)

2358 IID = Intrinsic::x86_avx512_pmulhu_w_512;

2359 else

2361 } else if (Name.starts_with("pmaddw.d.")) {

2362 if (VecWidth == 128)

2363 IID = Intrinsic::x86_sse2_pmadd_wd;

2364 else if (VecWidth == 256)

2365 IID = Intrinsic::x86_avx2_pmadd_wd;

2366 else if (VecWidth == 512)

2367 IID = Intrinsic::x86_avx512_pmaddw_d_512;

2368 else

2370 } else if (Name.starts_with("pmaddubs.w.")) {

2371 if (VecWidth == 128)

2372 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;

2373 else if (VecWidth == 256)

2374 IID = Intrinsic::x86_avx2_pmadd_ub_sw;

2375 else if (VecWidth == 512)

2376 IID = Intrinsic::x86_avx512_pmaddubs_w_512;

2377 else

2379 } else if (Name.starts_with("packsswb.")) {

2380 if (VecWidth == 128)

2381 IID = Intrinsic::x86_sse2_packsswb_128;

2382 else if (VecWidth == 256)

2383 IID = Intrinsic::x86_avx2_packsswb;

2384 else if (VecWidth == 512)

2385 IID = Intrinsic::x86_avx512_packsswb_512;

2386 else

2388 } else if (Name.starts_with("packssdw.")) {

2389 if (VecWidth == 128)

2390 IID = Intrinsic::x86_sse2_packssdw_128;

2391 else if (VecWidth == 256)

2392 IID = Intrinsic::x86_avx2_packssdw;

2393 else if (VecWidth == 512)

2394 IID = Intrinsic::x86_avx512_packssdw_512;

2395 else

2397 } else if (Name.starts_with("packuswb.")) {

2398 if (VecWidth == 128)

2399 IID = Intrinsic::x86_sse2_packuswb_128;

2400 else if (VecWidth == 256)

2401 IID = Intrinsic::x86_avx2_packuswb;

2402 else if (VecWidth == 512)

2403 IID = Intrinsic::x86_avx512_packuswb_512;

2404 else

2406 } else if (Name.starts_with("packusdw.")) {

2407 if (VecWidth == 128)

2408 IID = Intrinsic::x86_sse41_packusdw;

2409 else if (VecWidth == 256)

2410 IID = Intrinsic::x86_avx2_packusdw;

2411 else if (VecWidth == 512)

2412 IID = Intrinsic::x86_avx512_packusdw_512;

2413 else

2415 } else if (Name.starts_with("vpermilvar.")) {

2416 if (VecWidth == 128 && EltWidth == 32)

2417 IID = Intrinsic::x86_avx_vpermilvar_ps;

2418 else if (VecWidth == 128 && EltWidth == 64)

2419 IID = Intrinsic::x86_avx_vpermilvar_pd;

2420 else if (VecWidth == 256 && EltWidth == 32)

2421 IID = Intrinsic::x86_avx_vpermilvar_ps_256;

2422 else if (VecWidth == 256 && EltWidth == 64)

2423 IID = Intrinsic::x86_avx_vpermilvar_pd_256;

2424 else if (VecWidth == 512 && EltWidth == 32)

2425 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;

2426 else if (VecWidth == 512 && EltWidth == 64)

2427 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;

2428 else

2430 } else if (Name == "cvtpd2dq.256") {

2431 IID = Intrinsic::x86_avx_cvt_pd2dq_256;

2432 } else if (Name == "cvtpd2ps.256") {

2433 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;

2434 } else if (Name == "cvttpd2dq.256") {

2435 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;

2436 } else if (Name == "cvttps2dq.128") {

2437 IID = Intrinsic::x86_sse2_cvttps2dq;

2438 } else if (Name == "cvttps2dq.256") {

2439 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;

2440 } else if (Name.starts_with("permvar.")) {

2442 if (VecWidth == 256 && EltWidth == 32 && IsFloat)

2443 IID = Intrinsic::x86_avx2_permps;

2444 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)

2445 IID = Intrinsic::x86_avx2_permd;

2446 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)

2447 IID = Intrinsic::x86_avx512_permvar_df_256;

2448 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)

2449 IID = Intrinsic::x86_avx512_permvar_di_256;

2450 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)

2451 IID = Intrinsic::x86_avx512_permvar_sf_512;

2452 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)

2453 IID = Intrinsic::x86_avx512_permvar_si_512;

2454 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)

2455 IID = Intrinsic::x86_avx512_permvar_df_512;

2456 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)

2457 IID = Intrinsic::x86_avx512_permvar_di_512;

2458 else if (VecWidth == 128 && EltWidth == 16)

2459 IID = Intrinsic::x86_avx512_permvar_hi_128;

2460 else if (VecWidth == 256 && EltWidth == 16)

2461 IID = Intrinsic::x86_avx512_permvar_hi_256;

2462 else if (VecWidth == 512 && EltWidth == 16)

2463 IID = Intrinsic::x86_avx512_permvar_hi_512;

2464 else if (VecWidth == 128 && EltWidth == 8)

2465 IID = Intrinsic::x86_avx512_permvar_qi_128;

2466 else if (VecWidth == 256 && EltWidth == 8)

2467 IID = Intrinsic::x86_avx512_permvar_qi_256;

2468 else if (VecWidth == 512 && EltWidth == 8)

2469 IID = Intrinsic::x86_avx512_permvar_qi_512;

2470 else

2472 } else if (Name.starts_with("dbpsadbw.")) {

2473 if (VecWidth == 128)

2474 IID = Intrinsic::x86_avx512_dbpsadbw_128;

2475 else if (VecWidth == 256)

2476 IID = Intrinsic::x86_avx512_dbpsadbw_256;

2477 else if (VecWidth == 512)

2478 IID = Intrinsic::x86_avx512_dbpsadbw_512;

2479 else

2481 } else if (Name.starts_with("pmultishift.qb.")) {

2482 if (VecWidth == 128)

2483 IID = Intrinsic::x86_avx512_pmultishift_qb_128;

2484 else if (VecWidth == 256)

2485 IID = Intrinsic::x86_avx512_pmultishift_qb_256;

2486 else if (VecWidth == 512)

2487 IID = Intrinsic::x86_avx512_pmultishift_qb_512;

2488 else

2490 } else if (Name.starts_with("conflict.")) {

2491 if (Name[9] == 'd' && VecWidth == 128)

2492 IID = Intrinsic::x86_avx512_conflict_d_128;

2493 else if (Name[9] == 'd' && VecWidth == 256)

2494 IID = Intrinsic::x86_avx512_conflict_d_256;

2495 else if (Name[9] == 'd' && VecWidth == 512)

2496 IID = Intrinsic::x86_avx512_conflict_d_512;

2497 else if (Name[9] == 'q' && VecWidth == 128)

2498 IID = Intrinsic::x86_avx512_conflict_q_128;

2499 else if (Name[9] == 'q' && VecWidth == 256)

2500 IID = Intrinsic::x86_avx512_conflict_q_256;

2501 else if (Name[9] == 'q' && VecWidth == 512)

2502 IID = Intrinsic::x86_avx512_conflict_q_512;

2503 else

2505 } else if (Name.starts_with("pavg.")) {

2506 if (Name[5] == 'b' && VecWidth == 128)

2507 IID = Intrinsic::x86_sse2_pavg_b;

2508 else if (Name[5] == 'b' && VecWidth == 256)

2509 IID = Intrinsic::x86_avx2_pavg_b;

2510 else if (Name[5] == 'b' && VecWidth == 512)

2511 IID = Intrinsic::x86_avx512_pavg_b_512;

2512 else if (Name[5] == 'w' && VecWidth == 128)

2513 IID = Intrinsic::x86_sse2_pavg_w;

2514 else if (Name[5] == 'w' && VecWidth == 256)

2515 IID = Intrinsic::x86_avx2_pavg_w;

2516 else if (Name[5] == 'w' && VecWidth == 512)

2517 IID = Intrinsic::x86_avx512_pavg_w_512;

2518 else

2520 } else

2521 return false;

2522

2524 Args.pop_back();

2525 Args.pop_back();

2526 Rep = Builder.CreateIntrinsic(IID, Args);

2527 unsigned NumArgs = CI.arg_size();

2530 return true;

2531}

2532

2533

2534

2536 size_t Pos;

2537 if (AsmStr->find("mov\tfp") == 0 &&

2538 AsmStr->find("objc_retainAutoreleaseReturnValue") != std:🧵:npos &&

2539 (Pos = AsmStr->find("# marker")) != std:🧵:npos) {

2540 AsmStr->replace(Pos, 1, ";");

2541 }

2542}

2543

2546 Value *Rep = nullptr;

2547

2548 if (Name == "abs.i" || Name == "abs.ll") {

2550 Value *Neg = Builder.CreateNeg(Arg, "neg");

2551 Value *Cmp = Builder.CreateICmpSGE(

2553 Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");

2554 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {

2555 Type *Ty = (Name == "abs.bf16")

2559 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);

2560 Rep = Builder.CreateBitCast(Abs, CI->getType());

2561 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {

2562 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz

2563 : Intrinsic::nvvm_fabs;

2564 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));

2565 } else if (Name.consume_front("ex2.approx.")) {

2566

2567 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz

2568 : Intrinsic::nvvm_ex2_approx;

2569 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));

2570 } else if (Name.starts_with("atomic.load.add.f32.p") ||

2571 Name.starts_with("atomic.load.add.f64.p")) {

2576 } else if (Name.starts_with("atomic.load.inc.32.p") ||

2577 Name.starts_with("atomic.load.dec.32.p")) {

2582 Rep = Builder.CreateAtomicRMW(Op, Ptr, Val, MaybeAlign(),

2584 } else if (Name.consume_front("max.") &&

2585 (Name == "s" || Name == "i" || Name == "ll" || Name == "us" ||

2586 Name == "ui" || Name == "ull")) {

2589 Value *Cmp = Name.starts_with("u")

2590 ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond")

2591 : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");

2592 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");

2593 } else if (Name.consume_front("min.") &&

2594 (Name == "s" || Name == "i" || Name == "ll" || Name == "us" ||

2595 Name == "ui" || Name == "ull")) {

2598 Value *Cmp = Name.starts_with("u")

2599 ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond")

2600 : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");

2601 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");

2602 } else if (Name == "clz.ll") {

2603

2605 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},

2606 {Arg, Builder.getFalse()},

2607 nullptr, "ctlz");

2608 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");

2609 } else if (Name == "popc.ll") {

2610

2611

2613 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},

2614 Arg, nullptr, "ctpop");

2615 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");

2616 } else if (Name == "h2f") {

2617 Rep = Builder.CreateIntrinsic(Intrinsic::convert_from_fp16,

2619 nullptr, "h2f");

2620 } else if (Name.consume_front("bitcast.") &&

2621 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||

2622 Name == "d2ll")) {

2624 } else if (Name == "rotate.b32") {

2627 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,

2628 {Arg, Arg, ShiftAmt});

2629 } else if (Name == "rotate.b64") {

2632 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);

2633 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,

2634 {Arg, Arg, ZExtShiftAmt});

2635 } else if (Name == "rotate.right.b64") {

2638 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);

2639 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,

2640 {Arg, Arg, ZExtShiftAmt});

2641 } else if (Name == "swap.lo.hi.b64") {

2644 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,

2645 {Arg, Arg, Builder.getInt64(32)});

2646 } else if ((Name.consume_front("ptr.gen.to.") &&

2649 Name.starts_with(".to.gen"))) {

2651 } else if (Name.consume_front("ldg.global")) {

2654

2655 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));

2656 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);

2658 LD->setMetadata(LLVMContext::MD_invariant_load, MD);

2659 return LD;

2660 } else if (Name == "tanh.approx.f32") {

2661

2664 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),

2665 FMF);

2666 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {

2668 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);

2669 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,

2670 {}, {Arg});

2671 } else if (Name == "barrier") {

2672 Rep = Builder.CreateIntrinsic(

2673 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},

2675 } else if (Name == "barrier.sync") {

2676 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},

2678 } else if (Name == "barrier.sync.cnt") {

2679 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},

2681 } else {

2684 ->getReturnType()->getScalarType()->isBFloatTy()) {

2688 for (size_t I = 0; I < NewFn->arg_size(); ++I) {

2692 Args.push_back(

2694 ? Builder.CreateBitCast(Arg, NewType)

2695 : Arg);

2696 }

2697 Rep = Builder.CreateCall(NewFn, Args);

2698 if (F->getReturnType()->isIntegerTy())

2699 Rep = Builder.CreateBitCast(Rep, F->getReturnType());

2700 }

2701 }

2702

2703 return Rep;

2704}

2705

2709 Value *Rep = nullptr;

2710

2711 if (Name.starts_with("sse4a.movnt.")) {

2716

2719

2720

2721

2722 Value *Extract =

2723 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");

2724

2725 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));

2726 SI->setMetadata(LLVMContext::MD_nontemporal, Node);

2727 } else if (Name.starts_with("avx.movnt.") ||

2728 Name.starts_with("avx512.storent.")) {

2733

2736

2737 StoreInst *SI = Builder.CreateAlignedStore(

2738 Arg1, Arg0,

2740 SI->setMetadata(LLVMContext::MD_nontemporal, Node);

2741 } else if (Name == "sse2.storel.dq") {

2744

2746 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");

2747 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);

2748 Builder.CreateAlignedStore(Elt, Arg0, Align(1));

2749 } else if (Name.starts_with("sse.storeu.") ||

2750 Name.starts_with("sse2.storeu.") ||

2751 Name.starts_with("avx.storeu.")) {

2754 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));

2755 } else if (Name == "avx512.mask.store.ss") {

2756 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));

2758 Mask, false);

2759 } else if (Name.starts_with("avx512.mask.store")) {

2760

2761 bool Aligned = Name[17] != 'u';

2764 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {

2765

2766

2767 bool CmpEq = Name[9] == 'e';

2770 Rep = Builder.CreateSExt(Rep, CI->getType(), "");

2771 } else if (Name.starts_with("avx512.broadcastm")) {

2777 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);

2778 Rep = Builder.CreateVectorSplat(NumElts, Rep);

2779 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {

2781 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);

2782 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);

2783 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);

2784 } else if (Name.starts_with("avx.sqrt.p") ||

2785 Name.starts_with("sse2.sqrt.p") ||

2786 Name.starts_with("sse.sqrt.p")) {

2787 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),

2788 {CI->getArgOperand(0)});

2789 } else if (Name.starts_with("avx512.mask.sqrt.p")) {

2793 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512

2794 : Intrinsic::x86_avx512_sqrt_pd_512;

2795

2797 Rep = Builder.CreateIntrinsic(IID, Args);

2798 } else {

2799 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),

2800 {CI->getArgOperand(0)});

2801 }

2802 Rep =

2804 } else if (Name.starts_with("avx512.ptestm") ||

2805 Name.starts_with("avx512.ptestnm")) {

2809 Rep = Builder.CreateAnd(Op0, Op1);

2815 Rep = Builder.CreateICmp(Pred, Rep, Zero);

2817 } else if (Name.starts_with("avx512.mask.pbroadcast")) {

2819 ->getNumElements();

2820 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));

2821 Rep =

2823 } else if (Name.starts_with("avx512.kunpck")) {

2827 int Indices[64];

2828 for (unsigned i = 0; i != NumElts; ++i)

2829 Indices[i] = i;

2830

2831

2832

2833 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));

2834 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));

2835

2836

2837 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));

2838 Rep = Builder.CreateBitCast(Rep, CI->getType());

2839 } else if (Name == "avx512.kand.w") {

2842 Rep = Builder.CreateAnd(LHS, RHS);

2843 Rep = Builder.CreateBitCast(Rep, CI->getType());

2844 } else if (Name == "avx512.kandn.w") {

2847 LHS = Builder.CreateNot(LHS);

2848 Rep = Builder.CreateAnd(LHS, RHS);

2849 Rep = Builder.CreateBitCast(Rep, CI->getType());

2850 } else if (Name == "avx512.kor.w") {

2853 Rep = Builder.CreateOr(LHS, RHS);

2854 Rep = Builder.CreateBitCast(Rep, CI->getType());

2855 } else if (Name == "avx512.kxor.w") {

2858 Rep = Builder.CreateXor(LHS, RHS);

2859 Rep = Builder.CreateBitCast(Rep, CI->getType());

2860 } else if (Name == "avx512.kxnor.w") {

2863 LHS = Builder.CreateNot(LHS);

2864 Rep = Builder.CreateXor(LHS, RHS);

2865 Rep = Builder.CreateBitCast(Rep, CI->getType());

2866 } else if (Name == "avx512.knot.w") {

2868 Rep = Builder.CreateNot(Rep);

2869 Rep = Builder.CreateBitCast(Rep, CI->getType());

2870 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {

2873 Rep = Builder.CreateOr(LHS, RHS);

2874 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());

2876 if (Name[14] == 'c')

2878 else

2880 Rep = Builder.CreateICmpEQ(Rep, C);

2881 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());

2882 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||

2883 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||

2884 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||

2885 Name == "sse.div.ss" || Name == "sse2.div.sd") {

2888 ConstantInt::get(I32Ty, 0));

2890 ConstantInt::get(I32Ty, 0));

2892 if (Name.contains(".add."))

2893 EltOp = Builder.CreateFAdd(Elt0, Elt1);

2894 else if (Name.contains(".sub."))

2895 EltOp = Builder.CreateFSub(Elt0, Elt1);

2896 else if (Name.contains(".mul."))

2897 EltOp = Builder.CreateFMul(Elt0, Elt1);

2898 else

2899 EltOp = Builder.CreateFDiv(Elt0, Elt1);

2900 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,

2901 ConstantInt::get(I32Ty, 0));

2902 } else if (Name.starts_with("avx512.mask.pcmp")) {

2903

2904 bool CmpEq = Name[16] == 'e';

2906 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {

2910 switch (VecWidth) {

2911 default:

2913 case 128:

2914 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;

2915 break;

2916 case 256:

2917 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;

2918 break;

2919 case 512:

2920 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;

2921 break;

2922 }

2923

2924 Rep =

2927 } else if (Name.starts_with("avx512.mask.fpclass.p")) {

2932 if (VecWidth == 128 && EltWidth == 32)

2933 IID = Intrinsic::x86_avx512_fpclass_ps_128;

2934 else if (VecWidth == 256 && EltWidth == 32)

2935 IID = Intrinsic::x86_avx512_fpclass_ps_256;

2936 else if (VecWidth == 512 && EltWidth == 32)

2937 IID = Intrinsic::x86_avx512_fpclass_ps_512;

2938 else if (VecWidth == 128 && EltWidth == 64)

2939 IID = Intrinsic::x86_avx512_fpclass_pd_128;

2940 else if (VecWidth == 256 && EltWidth == 64)

2941 IID = Intrinsic::x86_avx512_fpclass_pd_256;

2942 else if (VecWidth == 512 && EltWidth == 64)

2943 IID = Intrinsic::x86_avx512_fpclass_pd_512;

2944 else

2946

2947 Rep =

2950 } else if (Name.starts_with("avx512.cmp.p")) {

2952 Type *OpTy = Args[0]->getType();

2956 if (VecWidth == 128 && EltWidth == 32)

2957 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;

2958 else if (VecWidth == 256 && EltWidth == 32)

2959 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;

2960 else if (VecWidth == 512 && EltWidth == 32)

2961 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;

2962 else if (VecWidth == 128 && EltWidth == 64)

2963 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;

2964 else if (VecWidth == 256 && EltWidth == 64)

2965 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;

2966 else if (VecWidth == 512 && EltWidth == 64)

2967 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;

2968 else

2970

2972 if (VecWidth == 512)

2974 Args.push_back(Mask);

2975

2976 Rep = Builder.CreateIntrinsic(IID, Args);

2977 } else if (Name.starts_with("avx512.mask.cmp.")) {

2978

2981 } else if (Name.starts_with("avx512.mask.ucmp.")) {

2984 } else if (Name.starts_with("avx512.cvtb2mask.") ||

2985 Name.starts_with("avx512.cvtw2mask.") ||

2986 Name.starts_with("avx512.cvtd2mask.") ||

2987 Name.starts_with("avx512.cvtq2mask.")) {

2992 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||

2993 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||

2994 Name.starts_with("avx512.mask.pabs")) {

2996 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||

2997 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||

2998 Name.starts_with("avx512.mask.pmaxs")) {

3000 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||

3001 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||

3002 Name.starts_with("avx512.mask.pmaxu")) {

3004 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||

3005 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||

3006 Name.starts_with("avx512.mask.pmins")) {

3008 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||

3009 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||

3010 Name.starts_with("avx512.mask.pminu")) {

3012 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||

3013 Name == "avx512.pmulu.dq.512" ||

3014 Name.starts_with("avx512.mask.pmulu.dq.")) {

3015 Rep = upgradePMULDQ(Builder, *CI, false);

3016 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||

3017 Name == "avx512.pmul.dq.512" ||

3018 Name.starts_with("avx512.mask.pmul.dq.")) {

3019 Rep = upgradePMULDQ(Builder, *CI, true);

3020 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||

3021 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {

3022 Rep =

3026 } else if (Name == "avx512.cvtusi2sd") {

3027 Rep =

3031 } else if (Name == "sse2.cvtss2sd") {

3033 Rep = Builder.CreateFPExt(

3036 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||

3037 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||

3038 Name.starts_with("avx512.mask.cvtdq2pd.") ||

3039 Name.starts_with("avx512.mask.cvtudq2pd.") ||

3040 Name.starts_with("avx512.mask.cvtdq2ps.") ||

3041 Name.starts_with("avx512.mask.cvtudq2ps.") ||

3042 Name.starts_with("avx512.mask.cvtqq2pd.") ||

3043 Name.starts_with("avx512.mask.cvtuqq2pd.") ||

3044 Name == "avx512.mask.cvtqq2ps.256" ||

3045 Name == "avx512.mask.cvtqq2ps.512" ||

3046 Name == "avx512.mask.cvtuqq2ps.256" ||

3047 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||

3048 Name == "avx.cvt.ps2.pd.256" ||

3049 Name == "avx512.mask.cvtps2pd.128" ||

3050 Name == "avx512.mask.cvtps2pd.256") {

3054

3055 unsigned NumDstElts = DstTy->getNumElements();

3057 assert(NumDstElts == 2 && "Unexpected vector size");

3058 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef{0, 1});

3059 }

3060

3061 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();

3062 bool IsUnsigned = Name.contains("cvtu");

3063 if (IsPS2PD)

3064 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");

3065 else if (CI->arg_size() == 4 &&

3068 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round

3069 : Intrinsic::x86_avx512_sitofp_round;

3070 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},

3072 } else {

3073 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")

3074 : Builder.CreateSIToFP(Rep, DstTy, "cvt");

3075 }

3076

3080 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||

3081 Name.starts_with("vcvtph2ps.")) {

3085 unsigned NumDstElts = DstTy->getNumElements();

3086 if (NumDstElts != SrcTy->getNumElements()) {

3087 assert(NumDstElts == 4 && "Unexpected vector size");

3088 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef{0, 1, 2, 3});

3089 }

3090 Rep = Builder.CreateBitCast(

3092 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");

3096 } else if (Name.starts_with("avx512.mask.load")) {

3097

3098 bool Aligned = Name[16] != 'u';

3101 } else if (Name.starts_with("avx512.mask.expand.load.")) {

3104 ResultTy->getNumElements());

3105

3106 Rep = Builder.CreateIntrinsic(

3107 Intrinsic::masked_expandload, ResultTy,

3109 } else if (Name.starts_with("avx512.mask.compress.store.")) {

3111 Value *MaskVec =

3114

3115 Rep = Builder.CreateIntrinsic(

3116 Intrinsic::masked_compressstore, ResultTy,

3118 } else if (Name.starts_with("avx512.mask.compress.") ||

3119 Name.starts_with("avx512.mask.expand.")) {

3121

3123 ResultTy->getNumElements());

3124

3125 bool IsCompress = Name[12] == 'c';

3126 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress

3127 : Intrinsic::x86_avx512_mask_expand;

3128 Rep = Builder.CreateIntrinsic(

3130 } else if (Name.starts_with("xop.vpcom")) {

3131 bool IsSigned;

3132 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||

3133 Name.ends_with("uq"))

3134 IsSigned = false;

3135 else if (Name.ends_with("b") || Name.ends_with("w") ||

3136 Name.ends_with("d") || Name.ends_with("q"))

3137 IsSigned = true;

3138 else

3140

3141 unsigned Imm;

3144 } else {

3145 Name = Name.substr(9);

3146 if (Name.starts_with("lt"))

3147 Imm = 0;

3148 else if (Name.starts_with("le"))

3149 Imm = 1;

3150 else if (Name.starts_with("gt"))

3151 Imm = 2;

3152 else if (Name.starts_with("ge"))

3153 Imm = 3;

3154 else if (Name.starts_with("eq"))

3155 Imm = 4;

3156 else if (Name.starts_with("ne"))

3157 Imm = 5;

3158 else if (Name.starts_with("false"))

3159 Imm = 6;

3160 else if (Name.starts_with("true"))

3161 Imm = 7;

3162 else

3164 }

3165

3167 } else if (Name.starts_with("xop.vpcmov")) {

3169 Value *NotSel = Builder.CreateNot(Sel);

3172 Rep = Builder.CreateOr(Sel0, Sel1);

3173 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||

3174 Name.starts_with("avx512.mask.prol")) {

3176 } else if (Name.starts_with("avx512.pror") ||

3177 Name.starts_with("avx512.mask.pror")) {

3179 } else if (Name.starts_with("avx512.vpshld.") ||

3180 Name.starts_with("avx512.mask.vpshld") ||

3181 Name.starts_with("avx512.maskz.vpshld")) {

3182 bool ZeroMask = Name[11] == 'z';

3184 } else if (Name.starts_with("avx512.vpshrd.") ||

3185 Name.starts_with("avx512.mask.vpshrd") ||

3186 Name.starts_with("avx512.maskz.vpshrd")) {

3187 bool ZeroMask = Name[11] == 'z';

3189 } else if (Name == "sse42.crc32.64.8") {

3192 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,

3194 Rep = Builder.CreateZExt(Rep, CI->getType(), "");

3195 } else if (Name.starts_with("avx.vbroadcast.s") ||

3196 Name.starts_with("avx512.vbroadcast.s")) {

3197

3199 Type *EltTy = VecTy->getElementType();

3200 unsigned EltNum = VecTy->getNumElements();

3204 for (unsigned I = 0; I < EltNum; ++I)

3205 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));

3206 } else if (Name.starts_with("sse41.pmovsx") ||

3207 Name.starts_with("sse41.pmovzx") ||

3208 Name.starts_with("avx2.pmovsx") ||

3209 Name.starts_with("avx2.pmovzx") ||

3210 Name.starts_with("avx512.mask.pmovsx") ||

3211 Name.starts_with("avx512.mask.pmovzx")) {

3213 unsigned NumDstElts = DstTy->getNumElements();

3214

3215

3217 for (unsigned i = 0; i != NumDstElts; ++i)

3218 ShuffleMask[i] = i;

3219

3220 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);

3221

3222 bool DoSext = Name.contains("pmovsx");

3223 Rep =

3224 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);

3225

3229 } else if (Name == "avx512.mask.pmov.qd.256" ||

3230 Name == "avx512.mask.pmov.qd.512" ||

3231 Name == "avx512.mask.pmov.wb.256" ||

3232 Name == "avx512.mask.pmov.wb.512") {

3234 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);

3235 Rep =

3237 } else if (Name.starts_with("avx.vbroadcastf128") ||

3238 Name == "avx2.vbroadcasti128") {

3239

3244 if (NumSrcElts == 2)

3245 Rep = Builder.CreateShuffleVector(Load, ArrayRef{0, 1, 0, 1});

3246 else

3247 Rep = Builder.CreateShuffleVector(Load,

3249 } else if (Name.starts_with("avx512.mask.shuf.i") ||

3250 Name.starts_with("avx512.mask.shuf.f")) {

3255 unsigned ControlBitsMask = NumLanes - 1;

3256 unsigned NumControlBits = NumLanes / 2;

3258

3259 for (unsigned l = 0; l != NumLanes; ++l) {

3260 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;

3261

3262 if (l >= NumLanes / 2)

3263 LaneMask += NumLanes;

3264 for (unsigned i = 0; i != NumElementsInLane; ++i)

3265 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);

3266 }

3267 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),

3269 Rep =

3271 } else if (Name.starts_with("avx512.mask.broadcastf") ||

3272 Name.starts_with("avx512.mask.broadcasti")) {

3274 ->getNumElements();

3275 unsigned NumDstElts =

3277

3279 for (unsigned i = 0; i != NumDstElts; ++i)

3280 ShuffleMask[i] = i % NumSrcElts;

3281

3282 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),

3284 Rep =

3286 } else if (Name.starts_with("avx2.pbroadcast") ||

3287 Name.starts_with("avx2.vbroadcast") ||

3288 Name.starts_with("avx512.pbroadcast") ||

3289 Name.starts_with("avx512.mask.broadcast.s")) {

3290

3296 Rep = Builder.CreateShuffleVector(Op, M);

3297

3301 } else if (Name.starts_with("sse2.padds.") ||

3302 Name.starts_with("avx2.padds.") ||

3303 Name.starts_with("avx512.padds.") ||

3304 Name.starts_with("avx512.mask.padds.")) {

3306 } else if (Name.starts_with("sse2.psubs.") ||

3307 Name.starts_with("avx2.psubs.") ||

3308 Name.starts_with("avx512.psubs.") ||

3309 Name.starts_with("avx512.mask.psubs.")) {

3311 } else if (Name.starts_with("sse2.paddus.") ||

3312 Name.starts_with("avx2.paddus.") ||

3313 Name.starts_with("avx512.mask.paddus.")) {

3315 } else if (Name.starts_with("sse2.psubus.") ||

3316 Name.starts_with("avx2.psubus.") ||

3317 Name.starts_with("avx512.mask.psubus.")) {

3319 } else if (Name.starts_with("avx512.mask.palignr.")) {

3323 false);

3324 } else if (Name.starts_with("avx512.mask.valign.")) {

3328 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {

3329

3332 Shift / 8);

3333 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {

3334

3337 Shift / 8);

3338 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||

3339 Name == "avx512.psll.dq.512") {

3340

3343 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||

3344 Name == "avx512.psrl.dq.512") {

3345

3348 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||

3349 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||

3350 Name.starts_with("avx2.pblendd.")) {

3355 unsigned NumElts = VecTy->getNumElements();

3356

3358 for (unsigned i = 0; i != NumElts; ++i)

3359 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;

3360

3361 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

3362 } else if (Name.starts_with("avx.vinsertf128.") ||

3363 Name == "avx2.vinserti128" ||

3364 Name.starts_with("avx512.mask.insert")) {

3368 unsigned DstNumElts =

3370 unsigned SrcNumElts =

3372 unsigned Scale = DstNumElts / SrcNumElts;

3373

3374

3375 Imm = Imm % Scale;

3376

3377

3379 for (unsigned i = 0; i != SrcNumElts; ++i)

3380 Idxs[i] = i;

3381 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)

3382 Idxs[i] = SrcNumElts;

3383 Rep = Builder.CreateShuffleVector(Op1, Idxs);

3384

3385

3386

3387

3388

3389

3390

3391

3392

3393

3394

3395

3396

3397 for (unsigned i = 0; i != DstNumElts; ++i)

3398 Idxs[i] = i;

3399

3400 for (unsigned i = 0; i != SrcNumElts; ++i)

3401 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;

3402 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);

3403

3404

3408 } else if (Name.starts_with("avx.vextractf128.") ||

3409 Name == "avx2.vextracti128" ||

3410 Name.starts_with("avx512.mask.vextract")) {

3413 unsigned DstNumElts =

3415 unsigned SrcNumElts =

3417 unsigned Scale = SrcNumElts / DstNumElts;

3418

3419

3420 Imm = Imm % Scale;

3421

3422

3424 for (unsigned i = 0; i != DstNumElts; ++i) {

3425 Idxs[i] = i + (Imm * DstNumElts);

3426 }

3427 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

3428

3429

3433 } else if (Name.starts_with("avx512.mask.perm.df.") ||

3434 Name.starts_with("avx512.mask.perm.di.")) {

3438 unsigned NumElts = VecTy->getNumElements();

3439

3441 for (unsigned i = 0; i != NumElts; ++i)

3442 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);

3443

3444 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

3445

3449 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {

3450

3451

3452

3453

3454

3455

3456

3457

3459

3461 unsigned HalfSize = NumElts / 2;

3463

3464

3467

3468

3471

3472

3473 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;

3474 for (unsigned i = 0; i < HalfSize; ++i)

3475 ShuffleMask[i] = StartIndex + i;

3476

3477

3478 StartIndex = (Imm & 0x10) ? HalfSize : 0;

3479 for (unsigned i = 0; i < HalfSize; ++i)

3480 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;

3481

3482 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);

3483

3484 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||

3485 Name.starts_with("avx512.mask.vpermil.p") ||

3486 Name.starts_with("avx512.mask.pshuf.d.")) {

3490 unsigned NumElts = VecTy->getNumElements();

3491

3492 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();

3493 unsigned IdxMask = ((1 << IdxSize) - 1);

3494

3496

3497

3498

3499 for (unsigned i = 0; i != NumElts; ++i)

3500 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);

3501

3502 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

3503

3507 } else if (Name == "sse2.pshufl.w" ||

3508 Name.starts_with("avx512.mask.pshufl.w.")) {

3512

3514 for (unsigned l = 0; l != NumElts; l += 8) {

3515 for (unsigned i = 0; i != 4; ++i)

3516 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;

3517 for (unsigned i = 4; i != 8; ++i)

3518 Idxs[i + l] = i + l;

3519 }

3520

3521 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

3522

3526 } else if (Name == "sse2.pshufh.w" ||

3527 Name.starts_with("avx512.mask.pshufh.w.")) {

3531

3533 for (unsigned l = 0; l != NumElts; l += 8) {

3534 for (unsigned i = 0; i != 4; ++i)

3535 Idxs[i + l] = i + l;

3536 for (unsigned i = 0; i != 4; ++i)

3537 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;

3538 }

3539

3540 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

3541

3545 } else if (Name.starts_with("avx512.mask.shuf.p")) {

3550

3552 unsigned HalfLaneElts = NumLaneElts / 2;

3553

3555 for (unsigned i = 0; i != NumElts; ++i) {

3556

3557 Idxs[i] = i - (i % NumLaneElts);

3558

3559 if ((i % NumLaneElts) >= HalfLaneElts)

3560 Idxs[i] += NumElts;

3561

3562

3563 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);

3564 }

3565

3566 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

3567

3568 Rep =

3570 } else if (Name.starts_with("avx512.mask.movddup") ||

3571 Name.starts_with("avx512.mask.movshdup") ||

3572 Name.starts_with("avx512.mask.movsldup")) {

3576

3577 unsigned Offset = 0;

3578 if (Name.starts_with("avx512.mask.movshdup."))

3580

3582 for (unsigned l = 0; l != NumElts; l += NumLaneElts)

3583 for (unsigned i = 0; i != NumLaneElts; i += 2) {

3584 Idxs[i + l + 0] = i + l + Offset;

3585 Idxs[i + l + 1] = i + l + Offset;

3586 }

3587

3588 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

3589

3590 Rep =

3592 } else if (Name.starts_with("avx512.mask.punpckl") ||

3593 Name.starts_with("avx512.mask.unpckl.")) {

3598

3600 for (int l = 0; l != NumElts; l += NumLaneElts)

3601 for (int i = 0; i != NumLaneElts; ++i)

3602 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);

3603

3604 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

3605

3606 Rep =

3608 } else if (Name.starts_with("avx512.mask.punpckh") ||

3609 Name.starts_with("avx512.mask.unpckh.")) {

3614

3616 for (int l = 0; l != NumElts; l += NumLaneElts)

3617 for (int i = 0; i != NumLaneElts; ++i)

3618 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);

3619

3620 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

3621

3622 Rep =

3624 } else if (Name.starts_with("avx512.mask.and.") ||

3625 Name.starts_with("avx512.mask.pand.")) {

3628 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),

3629 Builder.CreateBitCast(CI->getArgOperand(1), ITy));

3630 Rep = Builder.CreateBitCast(Rep, FTy);

3631 Rep =

3633 } else if (Name.starts_with("avx512.mask.andn.") ||

3634 Name.starts_with("avx512.mask.pandn.")) {

3637 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));

3638 Rep = Builder.CreateAnd(Rep,

3639 Builder.CreateBitCast(CI->getArgOperand(1), ITy));

3640 Rep = Builder.CreateBitCast(Rep, FTy);

3641 Rep =

3643 } else if (Name.starts_with("avx512.mask.or.") ||

3644 Name.starts_with("avx512.mask.por.")) {

3647 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),

3648 Builder.CreateBitCast(CI->getArgOperand(1), ITy));

3649 Rep = Builder.CreateBitCast(Rep, FTy);

3650 Rep =

3652 } else if (Name.starts_with("avx512.mask.xor.") ||

3653 Name.starts_with("avx512.mask.pxor.")) {

3656 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),

3657 Builder.CreateBitCast(CI->getArgOperand(1), ITy));

3658 Rep = Builder.CreateBitCast(Rep, FTy);

3659 Rep =

3661 } else if (Name.starts_with("avx512.mask.padd.")) {

3663 Rep =

3665 } else if (Name.starts_with("avx512.mask.psub.")) {

3667 Rep =

3669 } else if (Name.starts_with("avx512.mask.pmull.")) {

3671 Rep =

3673 } else if (Name.starts_with("avx512.mask.add.p")) {

3674 if (Name.ends_with(".512")) {

3676 if (Name[17] == 's')

3677 IID = Intrinsic::x86_avx512_add_ps_512;

3678 else

3679 IID = Intrinsic::x86_avx512_add_pd_512;

3680

3681 Rep = Builder.CreateIntrinsic(

3682 IID,

3684 } else {

3686 }

3687 Rep =

3689 } else if (Name.starts_with("avx512.mask.div.p")) {

3690 if (Name.ends_with(".512")) {

3692 if (Name[17] == 's')

3693 IID = Intrinsic::x86_avx512_div_ps_512;

3694 else

3695 IID = Intrinsic::x86_avx512_div_pd_512;

3696

3697 Rep = Builder.CreateIntrinsic(

3698 IID,

3700 } else {

3702 }

3703 Rep =

3705 } else if (Name.starts_with("avx512.mask.mul.p")) {

3706 if (Name.ends_with(".512")) {

3708 if (Name[17] == 's')

3709 IID = Intrinsic::x86_avx512_mul_ps_512;

3710 else

3711 IID = Intrinsic::x86_avx512_mul_pd_512;

3712

3713 Rep = Builder.CreateIntrinsic(

3714 IID,

3716 } else {

3718 }

3719 Rep =

3721 } else if (Name.starts_with("avx512.mask.sub.p")) {

3722 if (Name.ends_with(".512")) {

3724 if (Name[17] == 's')

3725 IID = Intrinsic::x86_avx512_sub_ps_512;

3726 else

3727 IID = Intrinsic::x86_avx512_sub_pd_512;

3728

3729 Rep = Builder.CreateIntrinsic(

3730 IID,

3732 } else {

3734 }

3735 Rep =

3737 } else if ((Name.starts_with("avx512.mask.max.p") ||

3738 Name.starts_with("avx512.mask.min.p")) &&

3739 Name.drop_front(18) == ".512") {

3740 bool IsDouble = Name[17] == 'd';

3741 bool IsMin = Name[13] == 'i';

3743 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},

3744 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};

3746

3747 Rep = Builder.CreateIntrinsic(

3748 IID,

3750 Rep =

3752 } else if (Name.starts_with("avx512.mask.lzcnt.")) {

3753 Rep =

3754 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),

3755 {CI->getArgOperand(0), Builder.getInt1(false)});

3756 Rep =

3758 } else if (Name.starts_with("avx512.mask.psll")) {

3759 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');

3760 bool IsVariable = Name[16] == 'v';

3761 char Size = Name[16] == '.' ? Name[17]

3762 : Name[17] == '.' ? Name[18]

3763 : Name[18] == '.' ? Name[19]

3764 : Name[20];

3765

3767 if (IsVariable && Name[17] != '.') {

3768 if (Size == 'd' && Name[17] == '2')

3769 IID = Intrinsic::x86_avx2_psllv_q;

3770 else if (Size == 'd' && Name[17] == '4')

3771 IID = Intrinsic::x86_avx2_psllv_q_256;

3772 else if (Size == 's' && Name[17] == '4')

3773 IID = Intrinsic::x86_avx2_psllv_d;

3774 else if (Size == 's' && Name[17] == '8')

3775 IID = Intrinsic::x86_avx2_psllv_d_256;

3776 else if (Size == 'h' && Name[17] == '8')

3777 IID = Intrinsic::x86_avx512_psllv_w_128;

3778 else if (Size == 'h' && Name[17] == '1')

3779 IID = Intrinsic::x86_avx512_psllv_w_256;

3780 else if (Name[17] == '3' && Name[18] == '2')

3781 IID = Intrinsic::x86_avx512_psllv_w_512;

3782 else

3784 } else if (Name.ends_with(".128")) {

3785 if (Size == 'd')

3786 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d

3787 : Intrinsic::x86_sse2_psll_d;

3788 else if (Size == 'q')

3789 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q

3790 : Intrinsic::x86_sse2_psll_q;

3791 else if (Size == 'w')

3792 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w

3793 : Intrinsic::x86_sse2_psll_w;

3794 else

3796 } else if (Name.ends_with(".256")) {

3797 if (Size == 'd')

3798 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d

3799 : Intrinsic::x86_avx2_psll_d;

3800 else if (Size == 'q')

3801 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q

3802 : Intrinsic::x86_avx2_psll_q;

3803 else if (Size == 'w')

3804 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w

3805 : Intrinsic::x86_avx2_psll_w;

3806 else

3808 } else {

3809 if (Size == 'd')

3810 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512

3811 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512

3812 : Intrinsic::x86_avx512_psll_d_512;

3813 else if (Size == 'q')

3814 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512

3815 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512

3816 : Intrinsic::x86_avx512_psll_q_512;

3817 else if (Size == 'w')

3818 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512

3819 : Intrinsic::x86_avx512_psll_w_512;

3820 else

3822 }

3823

3825 } else if (Name.starts_with("avx512.mask.psrl")) {

3826 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');

3827 bool IsVariable = Name[16] == 'v';

3828 char Size = Name[16] == '.' ? Name[17]

3829 : Name[17] == '.' ? Name[18]

3830 : Name[18] == '.' ? Name[19]

3831 : Name[20];

3832

3834 if (IsVariable && Name[17] != '.') {

3835 if (Size == 'd' && Name[17] == '2')

3836 IID = Intrinsic::x86_avx2_psrlv_q;

3837 else if (Size == 'd' && Name[17] == '4')

3838 IID = Intrinsic::x86_avx2_psrlv_q_256;

3839 else if (Size == 's' && Name[17] == '4')

3840 IID = Intrinsic::x86_avx2_psrlv_d;

3841 else if (Size == 's' && Name[17] == '8')

3842 IID = Intrinsic::x86_avx2_psrlv_d_256;

3843 else if (Size == 'h' && Name[17] == '8')

3844 IID = Intrinsic::x86_avx512_psrlv_w_128;

3845 else if (Size == 'h' && Name[17] == '1')

3846 IID = Intrinsic::x86_avx512_psrlv_w_256;

3847 else if (Name[17] == '3' && Name[18] == '2')

3848 IID = Intrinsic::x86_avx512_psrlv_w_512;

3849 else

3851 } else if (Name.ends_with(".128")) {

3852 if (Size == 'd')

3853 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d

3854 : Intrinsic::x86_sse2_psrl_d;

3855 else if (Size == 'q')

3856 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q

3857 : Intrinsic::x86_sse2_psrl_q;

3858 else if (Size == 'w')

3859 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w

3860 : Intrinsic::x86_sse2_psrl_w;

3861 else

3863 } else if (Name.ends_with(".256")) {

3864 if (Size == 'd')

3865 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d

3866 : Intrinsic::x86_avx2_psrl_d;

3867 else if (Size == 'q')

3868 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q

3869 : Intrinsic::x86_avx2_psrl_q;

3870 else if (Size == 'w')

3871 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w

3872 : Intrinsic::x86_avx2_psrl_w;

3873 else

3875 } else {

3876 if (Size == 'd')

3877 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512

3878 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512

3879 : Intrinsic::x86_avx512_psrl_d_512;

3880 else if (Size == 'q')

3881 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512

3882 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512

3883 : Intrinsic::x86_avx512_psrl_q_512;

3884 else if (Size == 'w')

3885 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512

3886 : Intrinsic::x86_avx512_psrl_w_512;

3887 else

3889 }

3890

3892 } else if (Name.starts_with("avx512.mask.psra")) {

3893 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');

3894 bool IsVariable = Name[16] == 'v';

3895 char Size = Name[16] == '.' ? Name[17]

3896 : Name[17] == '.' ? Name[18]

3897 : Name[18] == '.' ? Name[19]

3898 : Name[20];

3899

3901 if (IsVariable && Name[17] != '.') {

3902 if (Size == 's' && Name[17] == '4')

3903 IID = Intrinsic::x86_avx2_psrav_d;

3904 else if (Size == 's' && Name[17] == '8')

3905 IID = Intrinsic::x86_avx2_psrav_d_256;

3906 else if (Size == 'h' && Name[17] == '8')

3907 IID = Intrinsic::x86_avx512_psrav_w_128;

3908 else if (Size == 'h' && Name[17] == '1')

3909 IID = Intrinsic::x86_avx512_psrav_w_256;

3910 else if (Name[17] == '3' && Name[18] == '2')

3911 IID = Intrinsic::x86_avx512_psrav_w_512;

3912 else

3914 } else if (Name.ends_with(".128")) {

3915 if (Size == 'd')

3916 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d

3917 : Intrinsic::x86_sse2_psra_d;

3918 else if (Size == 'q')

3919 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128

3920 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128

3921 : Intrinsic::x86_avx512_psra_q_128;

3922 else if (Size == 'w')

3923 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w

3924 : Intrinsic::x86_sse2_psra_w;

3925 else

3927 } else if (Name.ends_with(".256")) {

3928 if (Size == 'd')

3929 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d

3930 : Intrinsic::x86_avx2_psra_d;

3931 else if (Size == 'q')

3932 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256

3933 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256

3934 : Intrinsic::x86_avx512_psra_q_256;

3935 else if (Size == 'w')

3936 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w

3937 : Intrinsic::x86_avx2_psra_w;

3938 else

3940 } else {

3941 if (Size == 'd')

3942 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512

3943 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512

3944 : Intrinsic::x86_avx512_psra_d_512;

3945 else if (Size == 'q')

3946 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512

3947 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512

3948 : Intrinsic::x86_avx512_psra_q_512;

3949 else if (Size == 'w')

3950 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512

3951 : Intrinsic::x86_avx512_psra_w_512;

3952 else

3954 }

3955

3957 } else if (Name.starts_with("avx512.mask.move.s")) {

3959 } else if (Name.starts_with("avx512.cvtmask2")) {

3961 } else if (Name.ends_with(".movntdqa")) {

3964

3965 LoadInst *LI = Builder.CreateAlignedLoad(

3969 Rep = LI;

3970 } else if (Name.starts_with("fma.vfmadd.") ||

3971 Name.starts_with("fma.vfmsub.") ||

3972 Name.starts_with("fma.vfnmadd.") ||

3973 Name.starts_with("fma.vfnmsub.")) {

3974 bool NegMul = Name[6] == 'n';

3975 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';

3976 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';

3977

3980

3981 if (IsScalar) {

3982 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);

3983 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);

3984 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);

3985 }

3986

3987 if (NegMul && !IsScalar)

3988 Ops[0] = Builder.CreateFNeg(Ops[0]);

3989 if (NegMul && IsScalar)

3990 Ops[1] = Builder.CreateFNeg(Ops[1]);

3991 if (NegAcc)

3992 Ops[2] = Builder.CreateFNeg(Ops[2]);

3993

3994 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);

3995

3996 if (IsScalar)

3998 } else if (Name.starts_with("fma4.vfmadd.s")) {

4001

4002 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);

4003 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);

4004 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);

4005

4006 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);

4007

4010 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||

4011 Name.starts_with("avx512.maskz.vfmadd.s") ||

4012 Name.starts_with("avx512.mask3.vfmadd.s") ||

4013 Name.starts_with("avx512.mask3.vfmsub.s") ||

4014 Name.starts_with("avx512.mask3.vfnmsub.s")) {

4015 bool IsMask3 = Name[11] == '3';

4016 bool IsMaskZ = Name[11] == 'z';

4017

4018 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);

4019 bool NegMul = Name[2] == 'n';

4020 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';

4021

4025

4026 if (NegMul && (IsMask3 || IsMaskZ))

4027 A = Builder.CreateFNeg(A);

4028 if (NegMul && !(IsMask3 || IsMaskZ))

4029 B = Builder.CreateFNeg(B);

4030 if (NegAcc)

4031 C = Builder.CreateFNeg(C);

4032

4033 A = Builder.CreateExtractElement(A, (uint64_t)0);

4034 B = Builder.CreateExtractElement(B, (uint64_t)0);

4035 C = Builder.CreateExtractElement(C, (uint64_t)0);

4036

4040

4042 if (Name.back() == 'd')

4043 IID = Intrinsic::x86_avx512_vfmadd_f64;

4044 else

4045 IID = Intrinsic::x86_avx512_vfmadd_f32;

4046 Rep = Builder.CreateIntrinsic(IID, Ops);

4047 } else {

4048 Rep = Builder.CreateFMA(A, B, C);

4049 }

4050

4052 : IsMask3 ? C

4053 : A;

4054

4055

4056

4057 if (NegAcc && IsMask3)

4058 PassThru =

4060

4062 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,

4064 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||

4065 Name.starts_with("avx512.mask.vfnmadd.p") ||

4066 Name.starts_with("avx512.mask.vfnmsub.p") ||

4067 Name.starts_with("avx512.mask3.vfmadd.p") ||

4068 Name.starts_with("avx512.mask3.vfmsub.p") ||

4069 Name.starts_with("avx512.mask3.vfnmsub.p") ||

4070 Name.starts_with("avx512.maskz.vfmadd.p")) {

4071 bool IsMask3 = Name[11] == '3';

4072 bool IsMaskZ = Name[11] == 'z';

4073

4074 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);

4075 bool NegMul = Name[2] == 'n';

4076 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';

4077

4081

4082 if (NegMul && (IsMask3 || IsMaskZ))

4083 A = Builder.CreateFNeg(A);

4084 if (NegMul && !(IsMask3 || IsMaskZ))

4085 B = Builder.CreateFNeg(B);

4086 if (NegAcc)

4087 C = Builder.CreateFNeg(C);

4088

4093

4094 if (Name[Name.size() - 5] == 's')

4095 IID = Intrinsic::x86_avx512_vfmadd_ps_512;

4096 else

4097 IID = Intrinsic::x86_avx512_vfmadd_pd_512;

4098

4099 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});

4100 } else {

4101 Rep = Builder.CreateFMA(A, B, C);

4102 }

4103

4107

4109 } else if (Name.starts_with("fma.vfmsubadd.p")) {

4113 if (VecWidth == 128 && EltWidth == 32)

4114 IID = Intrinsic::x86_fma_vfmaddsub_ps;

4115 else if (VecWidth == 256 && EltWidth == 32)

4116 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;

4117 else if (VecWidth == 128 && EltWidth == 64)

4118 IID = Intrinsic::x86_fma_vfmaddsub_pd;

4119 else if (VecWidth == 256 && EltWidth == 64)

4120 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;

4121 else

4123

4126 Ops[2] = Builder.CreateFNeg(Ops[2]);

4127 Rep = Builder.CreateIntrinsic(IID, Ops);

4128 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||

4129 Name.starts_with("avx512.mask3.vfmaddsub.p") ||

4130 Name.starts_with("avx512.maskz.vfmaddsub.p") ||

4131 Name.starts_with("avx512.mask3.vfmsubadd.p")) {

4132 bool IsMask3 = Name[11] == '3';

4133 bool IsMaskZ = Name[11] == 'z';

4134

4135 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);

4136 bool IsSubAdd = Name[3] == 's';

4139

4140 if (Name[Name.size() - 5] == 's')

4141 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;

4142 else

4143 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;

4144

4147 if (IsSubAdd)

4148 Ops[2] = Builder.CreateFNeg(Ops[2]);

4149

4150 Rep = Builder.CreateIntrinsic(IID, Ops);

4151 } else {

4153

4156

4158 CI->getModule(), Intrinsic::fma, Ops[0]->getType());

4159 Value *Odd = Builder.CreateCall(FMA, Ops);

4160 Ops[2] = Builder.CreateFNeg(Ops[2]);

4161 Value *Even = Builder.CreateCall(FMA, Ops);

4162

4163 if (IsSubAdd)

4165

4167 for (int i = 0; i != NumElts; ++i)

4168 Idxs[i] = i + (i % 2) * NumElts;

4169

4170 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);

4171 }

4172

4176

4178 } else if (Name.starts_with("avx512.mask.pternlog.") ||

4179 Name.starts_with("avx512.maskz.pternlog.")) {

4180 bool ZeroMask = Name[11] == 'z';

4184 if (VecWidth == 128 && EltWidth == 32)

4185 IID = Intrinsic::x86_avx512_pternlog_d_128;

4186 else if (VecWidth == 256 && EltWidth == 32)

4187 IID = Intrinsic::x86_avx512_pternlog_d_256;

4188 else if (VecWidth == 512 && EltWidth == 32)

4189 IID = Intrinsic::x86_avx512_pternlog_d_512;

4190 else if (VecWidth == 128 && EltWidth == 64)

4191 IID = Intrinsic::x86_avx512_pternlog_q_128;

4192 else if (VecWidth == 256 && EltWidth == 64)

4193 IID = Intrinsic::x86_avx512_pternlog_q_256;

4194 else if (VecWidth == 512 && EltWidth == 64)

4195 IID = Intrinsic::x86_avx512_pternlog_q_512;

4196 else

4198

4201 Rep = Builder.CreateIntrinsic(IID, Args);

4205 } else if (Name.starts_with("avx512.mask.vpmadd52") ||

4206 Name.starts_with("avx512.maskz.vpmadd52")) {

4207 bool ZeroMask = Name[11] == 'z';

4208 bool High = Name[20] == 'h' || Name[21] == 'h';

4211 if (VecWidth == 128 && )

4212 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;

4213 else if (VecWidth == 256 && )

4214 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;

4215 else if (VecWidth == 512 && )

4216 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;

4217 else if (VecWidth == 128 && High)

4218 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;

4219 else if (VecWidth == 256 && High)

4220 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;

4221 else if (VecWidth == 512 && High)

4222 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;

4223 else

4225

4228 Rep = Builder.CreateIntrinsic(IID, Args);

4232 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||

4233 Name.starts_with("avx512.mask.vpermt2var.") ||

4234 Name.starts_with("avx512.maskz.vpermt2var.")) {

4235 bool ZeroMask = Name[11] == 'z';

4236 bool IndexForm = Name[17] == 'i';

4238 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||

4239 Name.starts_with("avx512.maskz.vpdpbusd.") ||

4240 Name.starts_with("avx512.mask.vpdpbusds.") ||

4241 Name.starts_with("avx512.maskz.vpdpbusds.")) {

4242 bool ZeroMask = Name[11] == 'z';

4243 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';

4246 if (VecWidth == 128 && !IsSaturating)

4247 IID = Intrinsic::x86_avx512_vpdpbusd_128;

4248 else if (VecWidth == 256 && !IsSaturating)

4249 IID = Intrinsic::x86_avx512_vpdpbusd_256;

4250 else if (VecWidth == 512 && !IsSaturating)

4251 IID = Intrinsic::x86_avx512_vpdpbusd_512;

4252 else if (VecWidth == 128 && IsSaturating)

4253 IID = Intrinsic::x86_avx512_vpdpbusds_128;

4254 else if (VecWidth == 256 && IsSaturating)

4255 IID = Intrinsic::x86_avx512_vpdpbusds_256;

4256 else if (VecWidth == 512 && IsSaturating)

4257 IID = Intrinsic::x86_avx512_vpdpbusds_512;

4258 else

4260

4263

4264

4265

4266

4267 if (Args[1]->getType()->isVectorTy() &&

4269 ->getElementType()

4270 ->isIntegerTy(32) &&

4271 Args[2]->getType()->isVectorTy() &&

4273 ->getElementType()

4274 ->isIntegerTy(32)) {

4275 Type *NewArgType = nullptr;

4276 if (VecWidth == 128)

4277 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);

4278 else if (VecWidth == 256)

4279 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);

4280 else if (VecWidth == 512)

4281 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);

4282 else

4284

4285 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);

4286 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);

4287 }

4288

4289 Rep = Builder.CreateIntrinsic(IID, Args);

4293 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||

4294 Name.starts_with("avx512.maskz.vpdpwssd.") ||

4295 Name.starts_with("avx512.mask.vpdpwssds.") ||

4296 Name.starts_with("avx512.maskz.vpdpwssds.")) {

4297 bool ZeroMask = Name[11] == 'z';

4298 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';

4301 if (VecWidth == 128 && !IsSaturating)

4302 IID = Intrinsic::x86_avx512_vpdpwssd_128;

4303 else if (VecWidth == 256 && !IsSaturating)

4304 IID = Intrinsic::x86_avx512_vpdpwssd_256;

4305 else if (VecWidth == 512 && !IsSaturating)

4306 IID = Intrinsic::x86_avx512_vpdpwssd_512;

4307 else if (VecWidth == 128 && IsSaturating)

4308 IID = Intrinsic::x86_avx512_vpdpwssds_128;

4309 else if (VecWidth == 256 && IsSaturating)

4310 IID = Intrinsic::x86_avx512_vpdpwssds_256;

4311 else if (VecWidth == 512 && IsSaturating)

4312 IID = Intrinsic::x86_avx512_vpdpwssds_512;

4313 else

4315

4318 Rep = Builder.CreateIntrinsic(IID, Args);

4322 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||

4323 Name == "addcarry.u32" || Name == "addcarry.u64" ||

4324 Name == "subborrow.u32" || Name == "subborrow.u64") {

4326 if (Name[0] == 'a' && Name.back() == '2')

4327 IID = Intrinsic::x86_addcarry_32;

4328 else if (Name[0] == 'a' && Name.back() == '4')

4329 IID = Intrinsic::x86_addcarry_64;

4330 else if (Name[0] == 's' && Name.back() == '2')

4331 IID = Intrinsic::x86_subborrow_32;

4332 else if (Name[0] == 's' && Name.back() == '4')

4333 IID = Intrinsic::x86_subborrow_64;

4334 else

4336

4337

4340 Value *NewCall = Builder.CreateIntrinsic(IID, Args);

4341

4342

4343 Value *Data = Builder.CreateExtractValue(NewCall, 1);

4345

4346 Value *CF = Builder.CreateExtractValue(NewCall, 0);

4347

4349 Rep = nullptr;

4350 } else if (Name.starts_with("avx512.mask.") &&

4352

4353 }

4354

4355 return Rep;

4356}

4357

4360 if (Name.starts_with("neon.bfcvt")) {

4361 if (Name.starts_with("neon.bfcvtn2")) {

4363 std::iota(LoMask.begin(), LoMask.end(), 0);

4365 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);

4366 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);

4369 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);

4370 } else if (Name.starts_with("neon.bfcvtn")) {

4372 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);

4373 Type *V4BF16 =

4375 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);

4376 dbgs() << "Trunc: " << *Trunc << "\n";

4377 return Builder.CreateShuffleVector(

4379 } else {

4380 return Builder.CreateFPTrunc(CI->getOperand(0),

4382 }

4383 } else if (Name.starts_with("sve.fcvt")) {

4386 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)

4387 .Case("sve.fcvtnt.bf16f32",

4388 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)

4392

4394

4395

4396

4399

4400 if (Args[1]->getType() != BadPredTy)

4402

4403 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,

4404 BadPredTy, Args[1]);

4405 Args[1] = Builder.CreateIntrinsic(

4406 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);

4407

4408 return Builder.CreateIntrinsic(NewID, Args, nullptr,

4410 }

4411

4413}

4414

4417 if (Name == "mve.vctp64.old") {

4418

4419

4420 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},

4422 nullptr, CI->getName());

4423 Value *C1 = Builder.CreateIntrinsic(

4424 Intrinsic::arm_mve_pred_v2i,

4425 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);

4426 return Builder.CreateIntrinsic(

4427 Intrinsic::arm_mve_pred_i2v,

4429 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||

4430 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||

4431 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||

4432 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||

4433 Name ==

4434 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||

4435 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||

4436 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||

4437 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||

4438 Name ==

4439 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||

4440 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||

4441 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||

4442 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||

4443 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||

4444 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||

4445 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||

4446 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {

4447 std::vector<Type *> Tys;

4450 switch (ID) {

4451 case Intrinsic::arm_mve_mull_int_predicated:

4452 case Intrinsic::arm_mve_vqdmull_predicated:

4453 case Intrinsic::arm_mve_vldr_gather_base_predicated:

4455 break;

4456 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:

4457 case Intrinsic::arm_mve_vstr_scatter_base_predicated:

4458 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:

4460 V2I1Ty};

4461 break;

4462 case Intrinsic::arm_mve_vldr_gather_offset_predicated:

4465 break;

4466 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:

4469 break;

4470 case Intrinsic::arm_cde_vcx1q_predicated:

4471 case Intrinsic::arm_cde_vcx1qa_predicated:

4472 case Intrinsic::arm_cde_vcx2q_predicated:

4473 case Intrinsic::arm_cde_vcx2qa_predicated:

4474 case Intrinsic::arm_cde_vcx3q_predicated:

4475 case Intrinsic::arm_cde_vcx3qa_predicated:

4477 break;

4478 default:

4480 }

4481

4482 std::vector<Value *> Ops;

4484 Type *Ty = Op->getType();

4485 if (Ty->getScalarSizeInBits() == 1) {

4486 Value *C1 = Builder.CreateIntrinsic(

4487 Intrinsic::arm_mve_pred_v2i,

4489 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);

4490 }

4491 Ops.push_back(Op);

4492 }

4493

4494 return Builder.CreateIntrinsic(ID, Tys, Ops, nullptr,

4496 }

4497 llvm_unreachable("Unknown function for ARM CallBase upgrade.");

4498}

4499

4500

4501

4502

4503

4504

4520

4522 if (NumOperands < 3)

4523 return nullptr;

4524

4527 if (!PtrTy)

4528 return nullptr;

4529

4532 return nullptr;

4533

4535 bool IsVolatile = false;

4536

4537

4538

4539 if (NumOperands > 3)

4541

4542

4543

4544 if (NumOperands > 5) {

4546 IsVolatile = !VolatileArg || !VolatileArg->isZero();

4547 }

4548

4554

4556

4557

4560 if (VT->getElementType()->isIntegerTy(16)) {

4563 Val = Builder.CreateBitCast(Val, AsBF16);

4564 }

4565 }

4566

4567

4568

4569 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");

4571 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);

4572

4573 unsigned AddrSpace = PtrTy->getAddressSpace();

4576 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);

4578 RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);

4579 }

4580

4583 MDNode *RangeNotPrivate =

4586 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);

4587 }

4588

4589 if (IsVolatile)

4591

4592 return Builder.CreateBitCast(RMW, RetTy);

4593}

4594

4595

4596

4597

4602 Metadata *MD = MAV->getMetadata();

4604 }

4605 }

4606 return nullptr;

4607}

4608

4609

4613 return MAV->getMetadata();

4614 return nullptr;

4615}

4616

4618

4619

4620 return I->getDebugLoc().getAsMDNode();

4621}

4622

4623

4624

4625

4628 if (Name == "label") {

4631 } else if (Name == "assign") {

4636

4638 } else if (Name == "declare") {

4643 } else if (Name == "addr") {

4644

4646

4647

4650 }

4653 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr,

4655 } else if (Name == "value") {

4656

4657 unsigned VarOp = 1;

4658 unsigned ExprOp = 2;

4661

4663 return;

4664 VarOp = 2;

4665 ExprOp = 3;

4666 }

4671 }

4672 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");

4674}

4675

4676

4677

4679

4680

4681

4683 if ()

4684 return;

4685

4689

4690 if (!NewFn) {

4691

4693

4694 assert(Name.starts_with("llvm.") && "Intrinsic doesn't start with 'llvm.'");

4695 Name = Name.substr(5);

4696

4697 bool IsX86 = Name.consume_front("x86.");

4698 bool IsNVVM = Name.consume_front("nvvm.");

4699 bool IsAArch64 = Name.consume_front("aarch64.");

4700 bool IsARM = Name.consume_front("arm.");

4701 bool IsAMDGCN = Name.consume_front("amdgcn.");

4702 bool IsDbg = Name.consume_front("dbg.");

4703 Value *Rep = nullptr;

4704

4705 if (!IsX86 && Name == "stackprotectorcheck") {

4706 Rep = nullptr;

4707 } else if (IsNVVM) {

4709 } else if (IsX86) {

4711 } else if (IsAArch64) {

4713 } else if (IsARM) {

4715 } else if (IsAMDGCN) {

4717 } else if (IsDbg) {

4719 } else {

4721 }

4722

4723 if (Rep)

4726 return;

4727 }

4728

4729 const auto &DefaultCase = [&]() -> void {

4730 if (F == NewFn)

4731 return;

4732

4734

4737 "Unknown function for CallBase upgrade and isn't just a name change");

4739 return;

4740 }

4741

4742

4745 "Return type must have changed");

4746 assert(OldST->getNumElements() ==

4748 "Must have same number of elements");

4749

4751 CallInst *NewCI = Builder.CreateCall(NewFn, Args);

4754 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {

4755 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);

4756 Res = Builder.CreateInsertValue(Res, Elem, Idx);

4757 }

4760 return;

4761 }

4762

4763

4764

4767 return;

4768 };

4769 CallInst *NewCall = nullptr;

4771 default: {

4772 DefaultCase();

4773 return;

4774 }

4775 case Intrinsic::arm_neon_vst1:

4776 case Intrinsic::arm_neon_vst2:

4777 case Intrinsic::arm_neon_vst3:

4778 case Intrinsic::arm_neon_vst4:

4779 case Intrinsic::arm_neon_vst2lane:

4780 case Intrinsic::arm_neon_vst3lane:

4781 case Intrinsic::arm_neon_vst4lane: {

4783 NewCall = Builder.CreateCall(NewFn, Args);

4784 break;

4785 }

4786 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:

4787 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:

4788 case Intrinsic::aarch64_sve_bfdot_lane_v2: {

4789 LLVMContext &Ctx = F->getParent()->getContext();

4793 NewCall = Builder.CreateCall(NewFn, Args);

4794 break;

4795 }

4796 case Intrinsic::aarch64_sve_ld3_sret:

4797 case Intrinsic::aarch64_sve_ld4_sret:

4798 case Intrinsic::aarch64_sve_ld2_sret: {

4800 Name = Name.substr(5);

4807 unsigned MinElts = RetTy->getMinNumElements() / N;

4809 Value *NewLdCall = Builder.CreateCall(NewFn, Args);

4811 for (unsigned I = 0; I < N; I++) {

4812 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);

4813 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);

4814 }

4816 break;

4817 }

4818

4819 case Intrinsic::coro_end: {

4822 NewCall = Builder.CreateCall(NewFn, Args);

4823 break;

4824 }

4825

4826 case Intrinsic::vector_extract: {

4828 Name = Name.substr(5);

4829 if (!Name.starts_with("aarch64.sve.tuple.get")) {

4830 DefaultCase();

4831 return;

4832 }

4834 unsigned MinElts = RetTy->getMinNumElements();

4837 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});

4838 break;

4839 }

4840

4841 case Intrinsic::vector_insert: {

4843 Name = Name.substr(5);

4844 if (!Name.starts_with("aarch64.sve.tuple")) {

4845 DefaultCase();

4846 return;

4847 }

4848 if (Name.starts_with("aarch64.sve.tuple.set")) {

4853 NewCall = Builder.CreateCall(

4855 break;

4856 }

4857 if (Name.starts_with("aarch64.sve.tuple.create")) {

4859 .StartsWith("aarch64.sve.tuple.create2", 2)

4860 .StartsWith("aarch64.sve.tuple.create3", 3)

4861 .StartsWith("aarch64.sve.tuple.create4", 4)

4863 assert(N > 1 && "Create is expected to be between 2-4");

4866 unsigned MinElts = RetTy->getMinNumElements() / N;

4867 for (unsigned I = 0; I < N; I++) {

4869 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);

4870 }

4872 }

4873 break;

4874 }

4875

4876 case Intrinsic::arm_neon_bfdot:

4877 case Intrinsic::arm_neon_bfmmla:

4878 case Intrinsic::arm_neon_bfmlalb:

4879 case Intrinsic::arm_neon_bfmlalt:

4880 case Intrinsic::aarch64_neon_bfdot:

4881 case Intrinsic::aarch64_neon_bfmmla:

4882 case Intrinsic::aarch64_neon_bfmlalb:

4883 case Intrinsic::aarch64_neon_bfmlalt: {

4886 "Mismatch between function args and call args");

4887 size_t OperandWidth =

4889 assert((OperandWidth == 64 || OperandWidth == 128) &&

4890 "Unexpected operand width");

4892 auto Iter = CI->args().begin();

4893 Args.push_back(*Iter++);

4894 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));

4895 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));

4896 NewCall = Builder.CreateCall(NewFn, Args);

4897 break;

4898 }

4899

4900 case Intrinsic::bitreverse:

4901 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});

4902 break;

4903

4904 case Intrinsic::ctlz:

4905 case Intrinsic::cttz:

4907 "Mismatch between function args and call args");

4908 NewCall =

4909 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});

4910 break;

4911

4912 case Intrinsic::objectsize: {

4913 Value *NullIsUnknownSize =

4917 NewCall = Builder.CreateCall(

4919 break;

4920 }

4921

4922 case Intrinsic::ctpop:

4923 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});

4924 break;

4925

4926 case Intrinsic::convert_from_fp16:

4927 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});

4928 break;

4929

4930 case Intrinsic::dbg_value: {

4932 Name = Name.substr(5);

4933

4934 if (Name.starts_with("dbg.addr")) {

4938 NewCall =

4941 break;

4942 }

4943

4944

4946

4948 if (Offset->isZeroValue()) {

4949 NewCall = Builder.CreateCall(

4950 NewFn,

4952 break;

4953 }

4955 return;

4956 }

4957

4958 case Intrinsic::ptr_annotation:

4959

4961 DefaultCase();

4962 return;

4963 }

4964

4965

4966 NewCall = Builder.CreateCall(

4967 NewFn,

4973 return;

4974

4975 case Intrinsic::var_annotation:

4976

4978 DefaultCase();

4979 return;

4980 }

4981

4982 NewCall = Builder.CreateCall(

4983 NewFn,

4989 return;

4990

4991 case Intrinsic::riscv_aes32dsi:

4992 case Intrinsic::riscv_aes32dsmi:

4993 case Intrinsic::riscv_aes32esi:

4994 case Intrinsic::riscv_aes32esmi:

4995 case Intrinsic::riscv_sm4ks:

4996 case Intrinsic::riscv_sm4ed: {

4997

4998

5001 return;

5002

5006 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());

5007 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());

5008 }

5009

5012

5013 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});

5014 Value *Res = NewCall;

5016 Res = Builder.CreateIntCast(NewCall, CI->getType(), true);

5020 return;

5021 }

5022 case Intrinsic::nvvm_mapa_shared_cluster: {

5023

5024 NewCall =

5026 Value *Res = NewCall;

5027 Res = Builder.CreateAddrSpaceCast(

5032 return;

5033 }

5034 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:

5035 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {

5036

5038 Args[0] = Builder.CreateAddrSpaceCast(

5040

5041 NewCall = Builder.CreateCall(NewFn, Args);

5045 return;

5046 }

5047 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d:

5048 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d:

5049 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d:

5050 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d:

5051 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d:

5052 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d:

5053 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d:

5054 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d: {

5056

5057

5058

5061 Args[0] = Builder.CreateAddrSpaceCast(

5063

5064

5065

5066

5067 size_t NumArgs = CI->arg_size();

5070 Args.push_back(ConstantInt::get(Builder.getInt32Ty(), 0));

5071

5072 NewCall = Builder.CreateCall(NewFn, Args);

5076 return;

5077 }

5078 case Intrinsic::riscv_sha256sig0:

5079 case Intrinsic::riscv_sha256sig1:

5080 case Intrinsic::riscv_sha256sum0:

5081 case Intrinsic::riscv_sha256sum1:

5082 case Intrinsic::riscv_sm3p0:

5083 case Intrinsic::riscv_sm3p1: {

5084

5085

5087 return;

5088

5090 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());

5091

5092 NewCall = Builder.CreateCall(NewFn, Arg);

5094 Builder.CreateIntCast(NewCall, CI->getType(), true);

5098 return;

5099 }

5100

5101 case Intrinsic::x86_xop_vfrcz_ss:

5102 case Intrinsic::x86_xop_vfrcz_sd:

5103 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});

5104 break;

5105

5106 case Intrinsic::x86_xop_vpermil2pd:

5107 case Intrinsic::x86_xop_vpermil2ps:

5108 case Intrinsic::x86_xop_vpermil2pd_256:

5109 case Intrinsic::x86_xop_vpermil2ps_256: {

5113 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);

5114 NewCall = Builder.CreateCall(NewFn, Args);

5115 break;

5116 }

5117

5118 case Intrinsic::x86_sse41_ptestc:

5119 case Intrinsic::x86_sse41_ptestz:

5120 case Intrinsic::x86_sse41_ptestnzc: {

5121

5122

5123

5124

5127 return;

5128

5129

5131

5133

5134 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");

5135 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");

5136

5137 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});

5138 break;

5139 }

5140

5141 case Intrinsic::x86_rdtscp: {

5142

5143

5145 return;

5146

5147 NewCall = Builder.CreateCall(NewFn);

5148

5149 Value *Data = Builder.CreateExtractValue(NewCall, 1);

5151

5152 Value *TSC = Builder.CreateExtractValue(NewCall, 0);

5153

5157 return;

5158 }

5159

5160 case Intrinsic::x86_sse41_insertps:

5161 case Intrinsic::x86_sse41_dppd:

5162 case Intrinsic::x86_sse41_dpps:

5163 case Intrinsic::x86_sse41_mpsadbw:

5164 case Intrinsic::x86_avx_dp_ps_256:

5165 case Intrinsic::x86_avx2_mpsadbw: {

5166

5167

5169

5170

5171 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");

5172 NewCall = Builder.CreateCall(NewFn, Args);

5173 break;

5174 }

5175

5176 case Intrinsic::x86_avx512_mask_cmp_pd_128:

5177 case Intrinsic::x86_avx512_mask_cmp_pd_256:

5178 case Intrinsic::x86_avx512_mask_cmp_pd_512:

5179 case Intrinsic::x86_avx512_mask_cmp_ps_128:

5180 case Intrinsic::x86_avx512_mask_cmp_ps_256:

5181 case Intrinsic::x86_avx512_mask_cmp_ps_512: {

5183 unsigned NumElts =

5185 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);

5186

5187 NewCall = Builder.CreateCall(NewFn, Args);

5189

5193 return;

5194 }

5195

5196 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:

5197 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:

5198 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:

5199 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:

5200 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:

5201 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {

5205 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)

5206 Args[1] = Builder.CreateBitCast(

5208

5209 NewCall = Builder.CreateCall(NewFn, Args);

5210 Value *Res = Builder.CreateBitCast(

5212

5216 return;

5217 }

5218 case Intrinsic::x86_avx512bf16_dpbf16ps_128:

5219 case Intrinsic::x86_avx512bf16_dpbf16ps_256:

5220 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{

5222 unsigned NumElts =

5224 Args[1] = Builder.CreateBitCast(

5226 Args[2] = Builder.CreateBitCast(

5228

5229 NewCall = Builder.CreateCall(NewFn, Args);

5230 break;

5231 }

5232

5233 case Intrinsic::thread_pointer: {

5234 NewCall = Builder.CreateCall(NewFn, {});

5235 break;

5236 }

5237

5238 case Intrinsic::memcpy:

5239 case Intrinsic::memmove:

5240 case Intrinsic::memset: {

5241

5242

5243

5244

5245

5246

5247

5249 DefaultCase();

5250 return;

5251 }

5252

5253

5256 NewCall = Builder.CreateCall(NewFn, Args);

5258 AttributeList NewAttrs = AttributeList::get(

5259 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),

5260 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),

5261 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});

5264

5266 MemCI->setDestAlignment(Align->getMaybeAlignValue());

5267

5269 MTI->setSourceAlignment(Align->getMaybeAlignValue());

5270 break;

5271 }

5272

5273 case Intrinsic::masked_load:

5274 case Intrinsic::masked_gather:

5275 case Intrinsic::masked_store:

5276 case Intrinsic::masked_scatter: {

5278 DefaultCase();

5279 return;

5280 }

5281

5282 auto GetMaybeAlign = [](Value *Op) {

5284 uint64_t Val = CI->getZExtValue();

5285 if (Val == 0)

5289 }

5291 };

5292 auto GetAlign = [&](Value *Op) {

5297 };

5298

5301 case Intrinsic::masked_load:

5302 NewCall = Builder.CreateMaskedLoad(

5305 break;

5306 case Intrinsic::masked_gather:

5307 NewCall = Builder.CreateMaskedGather(

5309 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),

5312 break;

5313 case Intrinsic::masked_store:

5314 NewCall = Builder.CreateMaskedStore(

5317 break;

5318 case Intrinsic::masked_scatter:

5319 NewCall = Builder.CreateMaskedScatter(

5321 DL.getValueOrABITypeAlignment(

5325 break;

5326 default:

5328 }

5329

5332 break;

5333 }

5334

5335 case Intrinsic::lifetime_start:

5336 case Intrinsic::lifetime_end: {

5338 DefaultCase();

5339 return;

5340 }

5341

5343

5346

5347 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)

5348 NewCall = Builder.CreateLifetimeStart(Ptr);

5349 else

5350 NewCall = Builder.CreateLifetimeEnd(Ptr);

5351 break;

5352 }

5353

5354

5356 return;

5357 }

5358

5359 case Intrinsic::x86_avx512_vpdpbusd_128:

5360 case Intrinsic::x86_avx512_vpdpbusd_256:

5361 case Intrinsic::x86_avx512_vpdpbusd_512:

5362 case Intrinsic::x86_avx512_vpdpbusds_128:

5363 case Intrinsic::x86_avx512_vpdpbusds_256:

5364 case Intrinsic::x86_avx512_vpdpbusds_512:

5365 case Intrinsic::x86_avx2_vpdpbssd_128:

5366 case Intrinsic::x86_avx2_vpdpbssd_256:

5367 case Intrinsic::x86_avx10_vpdpbssd_512:

5368 case Intrinsic::x86_avx2_vpdpbssds_128:

5369 case Intrinsic::x86_avx2_vpdpbssds_256:

5370 case Intrinsic::x86_avx10_vpdpbssds_512:

5371 case Intrinsic::x86_avx2_vpdpbsud_128:

5372 case Intrinsic::x86_avx2_vpdpbsud_256:

5373 case Intrinsic::x86_avx10_vpdpbsud_512:

5374 case Intrinsic::x86_avx2_vpdpbsuds_128:

5375 case Intrinsic::x86_avx2_vpdpbsuds_256:

5376 case Intrinsic::x86_avx10_vpdpbsuds_512:

5377 case Intrinsic::x86_avx2_vpdpbuud_128:

5378 case Intrinsic::x86_avx2_vpdpbuud_256:

5379 case Intrinsic::x86_avx10_vpdpbuud_512:

5380 case Intrinsic::x86_avx2_vpdpbuuds_128:

5381 case Intrinsic::x86_avx2_vpdpbuuds_256:

5382 case Intrinsic::x86_avx10_vpdpbuuds_512: {

5386 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);

5387 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);

5388 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);

5389

5390 NewCall = Builder.CreateCall(NewFn, Args);

5391 break;

5392 }

5393 }

5394 assert(NewCall && "Should have either set this variable or returned through "

5395 "the default case");

5399}

5400

5402 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");

5403

5404

5405

5408

5409

5413

5414

5415 if (F != NewFn)

5416 F->eraseFromParent();

5417 }

5418}

5419

5422 if (NumOperands == 0)

5423 return &MD;

5424

5425

5427 return &MD;

5428

5430 if (NumOperands == 3) {

5433

5434 Metadata *Elts2[] = {ScalarType, ScalarType,

5439 }

5440

5444}

5445

5448 if (Opc != Instruction::BitCast)

5449 return nullptr;

5450

5451 Temp = nullptr;

5452 Type *SrcTy = V->getType();

5456

5457

5458

5461

5462 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);

5463 }

5464

5465 return nullptr;

5466}

5467

5469 if (Opc != Instruction::BitCast)

5470 return nullptr;

5471

5472 Type *SrcTy = C->getType();

5476

5477

5478

5480

5482 DestTy);

5483 }

5484

5485 return nullptr;

5486}

5487

5488

5489

5492 return false;

5493

5495

5496

5497

5499 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {

5500 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {

5501 if (Flag->getNumOperands() < 3)

5502 return false;

5503 if (MDString *K = dyn_cast_or_null(Flag->getOperand(1)))

5504 return K->getString() == "Debug Info Version";

5505 return false;

5506 });

5507 if (OpIt != ModFlags->op_end()) {

5508 const MDOperand &ValOp = (*OpIt)->getOperand(2);

5510 Version = CI->getZExtValue();

5511 }

5512 }

5513

5515 bool BrokenDebugInfo = false;

5518 if (!BrokenDebugInfo)

5519

5520 return false;

5521 else {

5522

5524 M.getContext().diagnose(Diag);

5525 }

5526 }

5529

5531 M.getContext().diagnose(DiagVersion);

5532 }

5534}

5535

5539

5541 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};

5542 unsigned Length = 0;

5543

5544 if (F->hasFnAttribute(Attr)) {

5545

5546

5547 StringRef S = F->getFnAttribute(Attr).getValueAsString();

5549 auto [Part, Rest] = S.split(',');

5551 S = Rest;

5552 }

5553 }

5554

5555 const unsigned Dim = DimC - 'x';

5556 assert(Dim < 3 && "Unexpected dim char");

5557

5559

5560

5561 const std::string VStr = llvm::utostr(VInt);

5562 Vect3[Dim] = VStr;

5564

5566 F->addFnAttr(Attr, NewAttr);

5567}

5568

5570 return S == "x" || S == "y" || S == "z";

5571}

5572

5575 if (K == "kernel") {

5578 return true;

5579 }

5580 if (K == "align") {

5581

5582

5583

5584

5585 const uint64_t AlignIdxValuePair =

5587 const unsigned Idx = (AlignIdxValuePair >> 16);

5588 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);

5591 return true;

5592 }

5593 if (K == "maxclusterrank" || K == "cluster_max_blocks") {

5596 return true;

5597 }

5598 if (K == "minctasm") {

5601 return true;

5602 }

5603 if (K == "maxnreg") {

5606 return true;

5607 }

5608 if (K.consume_front("maxntid") && isXYZ(K)) {

5610 return true;

5611 }

5612 if (K.consume_front("reqntid") && isXYZ(K)) {

5614 return true;

5615 }

5616 if (K.consume_front("cluster_dim_") && isXYZ(K)) {

5618 return true;

5619 }

5620 if (K == "grid_constant") {

5622 for (const auto &Op : cast(V)->operands()) {

5623

5624

5627 }

5628 return true;

5629 }

5630

5631 return false;

5632}

5633

5635 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");

5636 if (!NamedMD)

5637 return;

5638

5642 if (!SeenNodes.insert(MD).second)

5643 continue;

5644

5646 if (!GV)

5647 continue;

5648

5649 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");

5650

5652

5653

5654

5655

5656 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {

5658 const MDOperand &V = MD->getOperand(j + 1);

5660 if (!Upgraded)

5661 NewOperands.append({K, V});

5662 }

5663

5664 if (NewOperands.size() > 1)

5666 }

5667

5669 for (MDNode *N : NewNodes)

5671}

5672

5673

5674

5677 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";

5678 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);

5679 if (ModRetainReleaseMarker) {

5681 if (Op) {

5683 if (ID) {

5685 ID->getString().split(ValueComp, "#");

5686 if (ValueComp.size() == 2) {

5687 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();

5689 }

5691 M.eraseNamedMetadata(ModRetainReleaseMarker);

5693 }

5694 }

5695 }

5697}

5698

5700

5701

5702 auto UpgradeToIntrinsic = [&](const char *OldFunc,

5705

5706 if (!Fn)

5707 return;

5708

5711

5715 continue;

5716

5720

5721

5722

5726 continue;

5727

5728 bool InvalidCast = false;

5729

5730 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {

5732

5733

5734

5736

5737

5740 InvalidCast = true;

5741 break;

5742 }

5743 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));

5744 }

5745 Args.push_back(Arg);

5746 }

5747

5748 if (InvalidCast)

5749 continue;

5750

5751

5752 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);

5755

5756

5757 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());

5758

5762 }

5763

5766 };

5767

5768

5769

5770 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);

5771

5772

5773

5774

5776 return;

5777

5778 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {

5779 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},

5780 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},

5781 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},

5782 {"objc_autoreleaseReturnValue",

5783 llvm::Intrinsic::objc_autoreleaseReturnValue},

5784 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},

5785 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},

5786 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},

5787 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},

5788 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},

5789 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},

5790 {"objc_release", llvm::Intrinsic::objc_release},

5791 {"objc_retain", llvm::Intrinsic::objc_retain},

5792 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},

5793 {"objc_retainAutoreleaseReturnValue",

5794 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},

5795 {"objc_retainAutoreleasedReturnValue",

5796 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},

5797 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},

5798 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},

5799 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},

5800 {"objc_unsafeClaimAutoreleasedReturnValue",

5801 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},

5802 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},

5803 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},

5804 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},

5805 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},

5806 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},

5807 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},

5808 {"objc_arc_annotation_topdown_bbstart",

5809 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},

5810 {"objc_arc_annotation_topdown_bbend",

5811 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},

5812 {"objc_arc_annotation_bottomup_bbstart",

5813 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},

5814 {"objc_arc_annotation_bottomup_bbend",

5815 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};

5816

5817 for (auto &I : RuntimeFuncs)

5818 UpgradeToIntrinsic(I.first, I.second);

5819}

5820

5822 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();

5823 if (!ModFlags)

5824 return false;

5825

5826 bool HasObjCFlag = false, HasClassProperties = false, Changed = false;

5827 bool HasSwiftVersionFlag = false;

5828 uint8_t SwiftMajorVersion, SwiftMinorVersion;

5832

5833 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {

5835 if (Op->getNumOperands() != 3)

5836 continue;

5838 if ()

5839 continue;

5844 Op->getOperand(2)};

5847 };

5848

5849 if (ID->getString() == "Objective-C Image Info Version")

5850 HasObjCFlag = true;

5851 if (ID->getString() == "Objective-C Class Properties")

5852 HasClassProperties = true;

5853

5854 if (ID->getString() == "PIC Level") {

5855 if (auto *Behavior =

5857 uint64_t V = Behavior->getLimitedValue();

5860 }

5861 }

5862

5863 if (ID->getString() == "PIE Level")

5864 if (auto *Behavior =

5866 if (Behavior->getLimitedValue() == Module::Error)

5868

5869

5870

5871 if (ID->getString() == "branch-target-enforcement" ||

5872 ID->getString().starts_with("sign-return-address")) {

5873 if (auto *Behavior =

5875 if (Behavior->getLimitedValue() == Module::Error) {

5879 Op->getOperand(1), Op->getOperand(2)};

5882 }

5883 }

5884 }

5885

5886

5887

5888

5889 if (ID->getString() == "Objective-C Image Info Section") {

5892 Value->getString().split(ValueComp, " ");

5893 if (ValueComp.size() != 1) {

5894 std::string NewValue;

5895 for (auto &S : ValueComp)

5896 NewValue += S.str();

5897 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),

5901 }

5902 }

5903 }

5904

5905

5906

5907 if (ID->getString() == "Objective-C Garbage Collection") {

5909 if (Md) {

5910 assert(Md->getValue() && "Expected non-empty metadata");

5911 auto Type = Md->getValue()->getType();

5912 if (Type == Int8Ty)

5913 continue;

5914 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();

5915 if ((Val & 0xff) != Val) {

5916 HasSwiftVersionFlag = true;

5917 SwiftABIVersion = (Val & 0xff00) >> 8;

5918 SwiftMajorVersion = (Val & 0xff000000) >> 24;

5919 SwiftMinorVersion = (Val & 0xff0000) >> 16;

5920 }

5923 Op->getOperand(1),

5927 }

5928 }

5929

5930 if (ID->getString() == "amdgpu_code_object_version") {

5932 Op->getOperand(0),

5933 MDString::get(M.getContext(), "amdhsa_code_object_version"),

5934 Op->getOperand(2)};

5937 }

5938 }

5939

5940

5941

5942

5943

5944

5945 if (HasObjCFlag && !HasClassProperties) {

5949 }

5950

5951 if (HasSwiftVersionFlag) {

5952 M.addModuleFlag(Module::Error, "Swift ABI Version",

5953 SwiftABIVersion);

5954 M.addModuleFlag(Module::Error, "Swift Major Version",

5955 ConstantInt::get(Int8Ty, SwiftMajorVersion));

5956 M.addModuleFlag(Module::Error, "Swift Minor Version",

5957 ConstantInt::get(Int8Ty, SwiftMinorVersion));

5959 }

5960

5962}

5963

5965 auto TrimSpaces = [](StringRef Section) -> std::string {

5967 Section.split(Components, ',');

5968

5971

5972 for (auto Component : Components)

5973 OS << ',' << Component.trim();

5974

5975 return std::string(OS.str().substr(1));

5976 };

5977

5978 for (auto &GV : M.globals()) {

5979 if (!GV.hasSection())

5980 continue;

5981

5982 StringRef Section = GV.getSection();

5983

5984 if (!Section.starts_with("__DATA, __objc_catlist"))

5985 continue;

5986

5987

5988

5989 GV.setSection(TrimSpaces(Section));

5990 }

5991}

5992

5993namespace {

5994

5995

5996

5997

5998

5999

6000

6001

6002

6003

6004

6005struct StrictFPUpgradeVisitor : public InstVisitor {

6006 StrictFPUpgradeVisitor() = default;

6007

6009 if (.isStrictFP())

6010 return;

6012 return;

6013

6014

6015 Call.removeFnAttr(Attribute::StrictFP);

6016 Call.addFnAttr(Attribute::NoBuiltin);

6017 }

6018};

6019

6020

6021struct AMDGPUUnsafeFPAtomicsUpgradeVisitor

6022 : public InstVisitor {

6023 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;

6024

6025 void visitAtomicRMWInst(AtomicRMWInst &RMW) {

6027 return;

6028

6030 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);

6033 }

6034};

6035}

6036

6038

6039

6040 if (.isDeclaration() && .hasFnAttribute(Attribute::StrictFP)) {

6041 StrictFPUpgradeVisitor SFPV;

6042 SFPV.visit(F);

6043 }

6044

6045

6046 F.removeRetAttrs(AttributeFuncs::typeIncompatible(

6047 F.getReturnType(), F.getAttributes().getRetAttrs()));

6048 for (auto &Arg : F.args())

6049 Arg.removeAttrs(

6050 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));

6051

6052

6053

6054 if (Attribute A = F.getFnAttribute("implicit-section-name");

6055 A.isValid() && A.isStringAttribute()) {

6056 F.setSection(A.getValueAsString());

6057 F.removeFnAttr("implicit-section-name");

6058 }

6059

6060 if (.empty()) {

6061

6062

6063

6064 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");

6065 A.isValid()) {

6066

6067 if (A.getValueAsBool()) {

6068 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;

6069 Visitor.visit(F);

6070 }

6071

6072

6073

6074 F.removeFnAttr("amdgpu-unsafe-fp-atomics");

6075 }

6076 }

6077}

6078

6079

6082 if (.hasFnAttribute(FnAttrName))

6083 F.addFnAttr(FnAttrName, Value);

6084}

6085

6086

6087

6088

6090 if (.hasFnAttribute(FnAttrName)) {

6091 if (Set)

6092 F.addFnAttr(FnAttrName);

6093 } else {

6094 auto A = F.getFnAttribute(FnAttrName);

6095 if ("false" == A.getValueAsString())

6096 F.removeFnAttr(FnAttrName);

6097 else if ("true" == A.getValueAsString()) {

6098 F.removeFnAttr(FnAttrName);

6099 F.addFnAttr(FnAttrName);

6100 }

6101 }

6102}

6103

6105 Triple T(M.getTargetTriple());

6106 if (.isThumb() && .isARM() && .isAArch64())

6107 return;

6108

6115

6116 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();

6117 if (ModFlags) {

6118 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {

6120 if (Op->getNumOperands() != 3)

6121 continue;

6122

6125 if ( || !CI)

6126 continue;

6127

6129 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue

6130 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue

6131 : IDStr == "guarded-control-stack" ? &GCSValue

6132 : IDStr == "sign-return-address" ? &SRAValue

6133 : IDStr == "sign-return-address-all" ? &SRAALLValue

6134 : IDStr == "sign-return-address-with-bkey"

6135 ? &SRABKeyValue

6136 : nullptr;

6137 if (!ValPtr)

6138 continue;

6139

6140 *ValPtr = CI->getZExtValue();

6141 if (*ValPtr == 2)

6142 return;

6143 }

6144 }

6145

6146 bool BTE = BTEValue == 1;

6147 bool BPPLR = BPPLRValue == 1;

6148 bool GCS = GCSValue == 1;

6149 bool SRA = SRAValue == 1;

6150

6151 StringRef SignTypeValue = "non-leaf";

6152 if (SRA && SRAALLValue == 1)

6153 SignTypeValue = "all";

6154

6155 StringRef SignKeyValue = "a_key";

6156 if (SRA && SRABKeyValue == 1)

6157 SignKeyValue = "b_key";

6158

6159 for (Function &F : M.getFunctionList()) {

6160 if (F.isDeclaration())

6161 continue;

6162

6163 if (SRA) {

6166 } else {

6167 if (auto A = F.getFnAttribute("sign-return-address");

6168 A.isValid() && "none" == A.getValueAsString()) {

6169 F.removeFnAttr("sign-return-address");

6170 F.removeFnAttr("sign-return-address-key");

6171 }

6172 }

6176 }

6177

6178 if (BTE)

6179 M.setModuleFlag(llvm::Module::Min, "branch-target-enforcement", 2);

6180 if (BPPLR)

6181 M.setModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", 2);

6182 if (GCS)

6184 if (SRA) {

6186 if (SRAALLValue == 1)

6187 M.setModuleFlag(llvm::Module::Min, "sign-return-address-all", 2);

6188 if (SRABKeyValue == 1)

6189 M.setModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 2);

6190 }

6191}

6192

6195 if ()

6196 return false;

6197 if (T->getNumOperands() < 1)

6198 return false;

6200 if (!S)

6201 return false;

6202 return S->getString().starts_with("llvm.vectorizer.");

6203}

6204

6206 StringRef OldPrefix = "llvm.vectorizer.";

6208

6209 if (OldTag == "llvm.vectorizer.unroll")

6210 return MDString::get(C, "llvm.loop.interleave.count");

6211

6214 .str());

6215}

6216

6219 if ()

6220 return MD;

6221 if (T->getNumOperands() < 1)

6222 return MD;

6224 if (!OldTag)

6225 return MD;

6226 if (!OldTag->getString().starts_with("llvm.vectorizer."))

6227 return MD;

6228

6229

6231 Ops.reserve(T->getNumOperands());

6232 Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString()));

6233 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)

6234 Ops.push_back(T->getOperand(I));

6235

6237}

6238

6241 if ()

6242 return &N;

6243

6245 return &N;

6246

6248 Ops.reserve(T->getNumOperands());

6249 for (Metadata *MD : T->operands())

6251

6253}

6254

6257

6258

6259 if ((T.isSPIR() || (T.isSPIRV() && .isSPIRVLogical())) &&

6260 .contains("-G") && .starts_with("G")) {

6261 return DL.empty() ? std::string("G1") : (DL + "-G1").str();

6262 }

6263

6264 if (T.isLoongArch64() || T.isRISCV64()) {

6265

6266 auto I = DL.find("-n64-");

6268 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();

6269 return DL.str();

6270 }

6271

6272

6273 std::string Res = DL.str();

6274 if (T.isAMDGPU()) {

6275

6276 if (.contains("-G") && .starts_with("G"))

6277 Res.append(Res.empty() ? "G1" : "-G1");

6278

6279

6280 if (T.isAMDGCN()) {

6281

6282

6283

6284

6285 if (.contains("-ni") && .starts_with("ni"))

6286 Res.append("-ni:7:8:9");

6287

6288 if (DL.ends_with("ni:7"))

6289 Res.append(":8:9");

6290 if (DL.ends_with("ni:7:8"))

6291 Res.append(":9");

6292

6293

6294

6295 if (.contains("-p7") && .starts_with("p7"))

6296 Res.append("-p7:160:256:256:32");

6297 if (.contains("-p8") && .starts_with("p8"))

6298 Res.append("-p8:128:128:128:48");

6299 constexpr StringRef OldP8("-p8:128:128-");

6300 if (DL.contains(OldP8))

6301 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");

6302 if (.contains("-p9") && .starts_with("p9"))

6303 Res.append("-p9:192:256:256:32");

6304 }

6305

6306

6307 if (.contains("m:e"))

6308 Res = Res.empty() ? "m:e" : "m:e-" + Res;

6309

6310 return Res;

6311 }

6312

6313 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {

6314

6315

6316 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};

6317 if (.contains(AddrSpaces)) {

6319 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");

6320 if (R.match(Res, &Groups))

6321 Res = (Groups[1] + AddrSpaces + Groups[3]).str();

6322 }

6323 };

6324

6325

6326 if (T.isAArch64()) {

6327

6328 if (.empty() && .contains("-Fn32"))

6329 Res.append("-Fn32");

6330 AddPtr32Ptr64AddrSpaces();

6331 return Res;

6332 }

6333

6334 if (T.isSPARC() || (T.isMIPS64() && .contains("m:m")) || T.isPPC64() ||

6335 T.isWasm()) {

6336

6337

6338 std::string I64 = "-i64:64";

6339 std::string I128 = "-i128:128";

6341 size_t Pos = Res.find(I64);

6342 if (Pos != size_t(-1))

6343 Res.insert(Pos + I64.size(), I128);

6344 }

6345 return Res;

6346 }

6347

6348 if (.isX86())

6349 return Res;

6350

6351 AddPtr32Ptr64AddrSpaces();

6352

6353

6354

6355

6356

6357

6358

6359 if (.isOSIAMCU()) {

6360 std::string I128 = "-i128:128";

6363 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");

6364 if (R.match(Res, &Groups))

6366 }

6367 }

6368

6369

6370

6371

6372 if (T.isWindowsMSVCEnvironment() && .isArch64Bit()) {

6374 auto I = Ref.find("-f80:32-");

6376 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();

6377 }

6378

6379 return Res;

6380}

6381

6384 Attribute A = B.getAttribute("no-frame-pointer-elim");

6385 if (A.isValid()) {

6386

6387 FramePointer = A.getValueAsString() == "true" ? "all" : "none";

6388 B.removeAttribute("no-frame-pointer-elim");

6389 }

6390 if (B.contains("no-frame-pointer-elim-non-leaf")) {

6391

6392 if (FramePointer != "all")

6393 FramePointer = "non-leaf";

6394 B.removeAttribute("no-frame-pointer-elim-non-leaf");

6395 }

6396 if (!FramePointer.empty())

6397 B.addAttribute("frame-pointer", FramePointer);

6398

6399 A = B.getAttribute("null-pointer-is-valid");

6400 if (A.isValid()) {

6401

6402 bool NullPointerIsValid = A.getValueAsString() == "true";

6403 B.removeAttribute("null-pointer-is-valid");

6404 if (NullPointerIsValid)

6405 B.addAttribute(Attribute::NullPointerIsValid);

6406 }

6407}

6408

6410

6411

6412

6413

6415 return OBD.getTag() == "clang.arc.attachedcall" &&

6416 OBD.inputs().empty();

6417 });

6418}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

AMDGPU address space definition.

AMDGPU Register Bank Select

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

This file contains the simple types necessary to represent the attributes associated with functions a...

static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)

Definition AutoUpgrade.cpp:1960

static Metadata * upgradeLoopArgument(Metadata *MD)

Definition AutoUpgrade.cpp:6217

static bool isXYZ(StringRef S)

Definition AutoUpgrade.cpp:5569

static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)

Definition AutoUpgrade.cpp:1184

static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)

Definition AutoUpgrade.cpp:1800

static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)

Definition AutoUpgrade.cpp:1079

static bool upgradeRetainReleaseMarker(Module &M)

This checks for objc retain release marker which should be upgraded.

Definition AutoUpgrade.cpp:5675

static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)

Definition AutoUpgrade.cpp:2061

static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)

Definition AutoUpgrade.cpp:2287

static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)

Definition AutoUpgrade.cpp:2035

static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)

Definition AutoUpgrade.cpp:112

static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)

Definition AutoUpgrade.cpp:6080

static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)

Definition AutoUpgrade.cpp:1104

static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)

Definition AutoUpgrade.cpp:5573

static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)

Helper to unwrap intrinsic call MetadataAsValue operands.

Definition AutoUpgrade.cpp:4598

static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)

Definition AutoUpgrade.cpp:6205

static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)

Definition AutoUpgrade.cpp:5536

static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)

Definition AutoUpgrade.cpp:99

static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)

Definition AutoUpgrade.cpp:1915

static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)

Definition AutoUpgrade.cpp:2172

static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)

Definition AutoUpgrade.cpp:1886

static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)

Definition AutoUpgrade.cpp:4358

static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)

Definition AutoUpgrade.cpp:2273

static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)

Definition AutoUpgrade.cpp:515

static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)

Definition AutoUpgrade.cpp:2212

static bool consumeNVVMPtrAddrSpace(StringRef &Name)

Definition AutoUpgrade.cpp:1178

static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)

Definition AutoUpgrade.cpp:148

static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)

Definition AutoUpgrade.cpp:1834

static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name)

Definition AutoUpgrade.cpp:1032

static bool isOldLoopArgument(Metadata *MD)

Definition AutoUpgrade.cpp:6193

static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)

Definition AutoUpgrade.cpp:4415

static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)

Definition AutoUpgrade.cpp:83

static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)

Definition AutoUpgrade.cpp:4505

static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)

Definition AutoUpgrade.cpp:2151

static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)

Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.

Definition AutoUpgrade.cpp:4610

static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)

Definition AutoUpgrade.cpp:128

static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)

Definition AutoUpgrade.cpp:707

static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)

Definition AutoUpgrade.cpp:1866

static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)

Definition AutoUpgrade.cpp:1898

static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)

Definition AutoUpgrade.cpp:2100

static void rename(GlobalValue *GV)

Definition AutoUpgrade.cpp:63

static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)

Definition AutoUpgrade.cpp:67

static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)

Definition AutoUpgrade.cpp:138

static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))

static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)

Definition AutoUpgrade.cpp:2234

static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)

Definition AutoUpgrade.cpp:2020

static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)

Definition AutoUpgrade.cpp:2544

static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)

Definition AutoUpgrade.cpp:2266

static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)

Definition AutoUpgrade.cpp:2296

static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)

Convert debug intrinsic calls to non-instruction debug records.

Definition AutoUpgrade.cpp:4626

static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)

Definition AutoUpgrade.cpp:6089

static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)

Definition AutoUpgrade.cpp:2182

static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)

Definition AutoUpgrade.cpp:2133

static MDNode * getDebugLocSafe(const Instruction *I)

Definition AutoUpgrade.cpp:4617

static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)

Definition AutoUpgrade.cpp:2706

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

This file contains the declarations for the subclasses of Constant, which represent the different fla...

This file contains constants used for implementing Dwarf debug support.

Module.h This file contains the declarations for the Module class.

const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]

static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)

NVPTX address space definition.

static unsigned getNumElements(Type *Ty)

static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)

This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...

static SymbolRef::Type getType(const Symbol *Sym)

LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty

static const X86InstrFMA3Group Groups[]

Class for arbitrary precision integers.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

Class to represent array types.

static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)

This static method is the primary way to construct an ArrayType.

Type * getElementType() const

an instruction that atomically reads a memory location, combines it with another value,...

void setVolatile(bool V)

Specify whether this is a volatile RMW or not.

BinOp

This enumeration lists the possible modifications atomicrmw can make.

@ UIncWrap

Increment one up to a maximum value.

@ FMin

*p = minnum(old, v) minnum matches the behavior of llvm.minnum.

@ FMax

*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.

@ UDecWrap

Decrement one until a minimum value or zero.

bool isFloatingPointOperation() const

Functions, function parameters, and return types can have attributes to indicate how they should be t...

static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)

static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)

Return a uniquified Attribute object.

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

Function * getCalledFunction() const

Returns the function called, or null if this is an indirect function invocation or the function signa...

Value * getCalledOperand() const

void setAttributes(AttributeList A)

Set the attributes for this call.

Value * getArgOperand(unsigned i) const

FunctionType * getFunctionType() const

LLVM_ABI Intrinsic::ID getIntrinsicID() const

Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...

iterator_range< User::op_iterator > args()

Iteration adapter for range-for loops.

void setCalledOperand(Value *V)

unsigned arg_size() const

AttributeList getAttributes() const

Return the attributes for this call.

void setCalledFunction(Function *Fn)

Sets the function called, including updating the function type.

This class represents a function call, abstracting a target machine's calling convention.

void setTailCallKind(TailCallKind TCK)

static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)

Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...

static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)

This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.

Predicate

This enumeration lists the possible predicates for CmpInst subclasses.

@ ICMP_SLT

signed less than

@ ICMP_SLE

signed less or equal

@ ICMP_UGE

unsigned greater or equal

@ ICMP_UGT

unsigned greater than

@ ICMP_SGT

signed greater than

@ ICMP_ULT

unsigned less than

@ ICMP_SGE

signed greater or equal

@ ICMP_ULE

unsigned less or equal

static LLVM_ABI ConstantAggregateZero * get(Type *Ty)

static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)

static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)

static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)

Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.

static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)

This is the shared class of boolean and integer constants.

bool isZero() const

This is just a convenience method to make client code smaller for a common code.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

static LLVM_ABI ConstantPointerNull * get(PointerType *T)

Static factory methods - Return objects of the specified value.

static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)

static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)

Return the ConstantTokenNone.

This is an important base class in LLVM.

static LLVM_ABI Constant * getAllOnesValue(Type *Ty)

static LLVM_ABI Constant * getNullValue(Type *Ty)

Constructor to create a '0' constant of arbitrary type.

static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)

Append the opcodes Ops to DIExpr.

A parsed version of the target data layout string in and methods for querying it.

static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label, MDNode *DL)

For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.

Base class for non-instruction debug metadata records that have positions within IR.

static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression, MDNode *DI)

Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...

Convenience struct for specifying and reasoning about fast-math flags.

void setApproxFunc(bool B=true)

static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)

Class to represent function types.

Type * getParamType(unsigned i) const

Parameter type accessors.

Type * getReturnType() const

static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)

FunctionType * getFunctionType() const

Returns the FunctionType for me.

Intrinsic::ID getIntrinsicID() const LLVM_READONLY

getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...

const Function & getFunction() const

void eraseFromParent()

eraseFromParent - This method unlinks 'this' from the containing module and deletes it.

Type * getReturnType() const

Returns the type of the ret val.

Argument * getArg(unsigned i) const

LinkageTypes getLinkage() const

Type * getValueType() const

const Constant * getInitializer() const

getInitializer - Return the initializer for this global variable.

bool hasInitializer() const

Definitions have initializers, declarations don't.

PointerType * getPtrTy(unsigned AddrSpace=0)

Fetch the type representing a pointer.

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

Base class for instruction visitors.

const DebugLoc & getDebugLoc() const

Return the debug location for this node as a DebugLoc.

LLVM_ABI const Module * getModule() const

Return the module owning the function this instruction belongs to or nullptr it the function does not...

LLVM_ABI InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())

Copy metadata from SrcInst to this instruction.

LLVM_ABI const DataLayout & getDataLayout() const

Get the data layout of the module this instruction belongs to.

This is an important class for using LLVM in a threaded context.

An instruction for reading from memory.

LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)

Return metadata describing the range [Lo, Hi).

const MDOperand & getOperand(unsigned I) const

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

unsigned getNumOperands() const

Return number of MDNode operands.

LLVMContext & getContext() const

Tracking metadata reference owned by Metadata.

static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

A Module instance is used to store all the information related to an LLVM module.

ModFlagBehavior

This enumeration defines the supported behaviors of module flags.

@ Override

Uses the specified value, regardless of the behavior or value of the other module.

@ Error

Emits an error if two values disagree, otherwise the resulting value is that of the operands.

@ Min

Takes the min of the two values, which are required to be integers.

@ Max

Takes the max of the two values, which are required to be integers.

LLVM_ABI void setOperand(unsigned I, MDNode *New)

LLVM_ABI MDNode * getOperand(unsigned i) const

LLVM_ABI unsigned getNumOperands() const

LLVM_ABI void clearOperands()

Drop all references to this node's operands.

iterator_range< op_iterator > operands()

LLVM_ABI void addOperand(MDNode *M)

ArrayRef< InputTy > inputs() const

static LLVM_ABI PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const

matches - Match the regex against a given String.

static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)

ArrayRef< int > getShuffleMask() const

std::pair< iterator, bool > insert(PtrType Ptr)

Inserts Ptr if and only if there is no element in the container equal to Ptr.

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...

void append(ItTy in_start, ItTy in_end)

Add the specified range to the end of the SmallVector.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

An instruction for storing to memory.

A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...

StringRef - Represent a constant reference to a string, i.e.

std::pair< StringRef, StringRef > split(char Separator) const

Split into two substrings around the first occurrence of a separator character.

static constexpr size_t npos

constexpr StringRef substr(size_t Start, size_t N=npos) const

Return a reference to the substring from [Start, Start + N).

bool starts_with(StringRef Prefix) const

Check if this string starts with the given Prefix.

constexpr bool empty() const

empty - Check if the string is empty.

StringRef drop_front(size_t N=1) const

Return a StringRef equal to 'this' but with the first N elements dropped.

constexpr size_t size() const

size - Get the string size.

StringRef trim(char Char) const

Return string with consecutive Char characters starting from the left and right removed.

A switch()-like statement whose cases are string literals.

StringSwitch & Case(StringLiteral S, T Value)

StringSwitch & StartsWith(StringLiteral S, T Value)

StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)

Class to represent struct types.

static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)

This static method is the primary way to create a literal StructType.

unsigned getNumElements() const

Random access to the elements.

Type * getElementType(unsigned N) const

The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.

Triple - Helper class for working with autoconf configuration names.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)

bool isVectorTy() const

True if this is an instance of VectorType.

static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)

bool isFloatTy() const

Return true if this is 'float', a 32-bit IEEE fp type.

bool isBFloatTy() const

Return true if this is 'bfloat', a 16-bit bfloat type.

LLVM_ABI unsigned getPointerAddressSpace() const

Get the address space of this pointer or pointer vector type.

static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)

Type * getScalarType() const

If this is a vector type, return the element type, otherwise return 'this'.

LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY

Return the basic size of this type if it is a primitive type.

LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY

If this is a vector type, return the getPrimitiveSizeInBits value for the element type.

bool isPtrOrPtrVectorTy() const

Return true if this is a pointer type or a vector of pointer types.

bool isIntegerTy() const

True if this is an instance of IntegerType.

bool isFPOrFPVectorTy() const

Return true if this is a FP type or a vector of FP.

static LLVM_ABI Type * getFloatTy(LLVMContext &C)

static LLVM_ABI Type * getBFloatTy(LLVMContext &C)

static LLVM_ABI Type * getHalfTy(LLVMContext &C)

Value * getOperand(unsigned i) const

unsigned getNumOperands() const

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

LLVM_ABI void setName(const Twine &Name)

Change the name of the value.

LLVM_ABI void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

iterator_range< user_iterator > users()

LLVM_ABI const Value * stripPointerCasts() const

Strip off pointer casts, all-zero GEPs and address space casts.

LLVM_ABI LLVMContext & getContext() const

All values hold a context through their type.

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

LLVM_ABI void takeName(Value *V)

Transfer the name from V to this value.

Base class of all SIMD vector types.

static VectorType * getInteger(VectorType *VTy)

This static method gets a VectorType with the same number of elements as the input type,...

static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)

This static method is the primary way to construct an VectorType.

constexpr ScalarTy getFixedValue() const

const ParentTy * getParent() const

self_iterator getIterator()

A raw_ostream that writes to an SmallVector or SmallString.

StringRef str() const

Return a StringRef for the vector contents.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ LOCAL_ADDRESS

Address space for local memory.

@ FLAT_ADDRESS

Address space for flat memory.

@ PRIVATE_ADDRESS

Address space for private memory.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

@ PTX_Kernel

Call to a PTX kernel. Passes all arguments in parameter space.

@ C

The default llvm calling convention, compatible with C.

LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

LLVM_ABI void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl< IITDescriptor > &T)

Return the IIT table descriptor for the specified intrinsic into an array of IITDescriptors.

LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)

LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)

Return the attributes for an intrinsic.

LLVM_ABI bool getIntrinsicSignature(Intrinsic::ID, FunctionType *FT, SmallVectorImpl< Type * > &ArgTys)

Gets the type arguments of an intrinsic call by matching type contraints specified by the ....

@ ADDRESS_SPACE_SHARED_CLUSTER

std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)

Extract a Value from Metadata, if any, allowing null.

std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)

Extract a Value from Metadata, if any.

std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)

Extract a Value from Metadata.

This is an optimization pass for GlobalISel generic memory operations.

LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)

This is the complement to the above, replacing a specific call to an intrinsic function with a call t...

Definition AutoUpgrade.cpp:4678

LLVM_ABI void UpgradeSectionAttributes(Module &M)

Definition AutoUpgrade.cpp:5964

auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)

Get the size of a range.

LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)

Upgrade comment in call to inline asm that represents an objc retain release marker.

Definition AutoUpgrade.cpp:2535

bool isValidAtomicOrdering(Int I)

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty

LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)

This is a more granular function that simply checks an intrinsic function for upgrading,...

Definition AutoUpgrade.cpp:1748

LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)

Upgrade the loop attachment metadata node.

Definition AutoUpgrade.cpp:6239

auto dyn_cast_if_present(const Y &Val)

dyn_cast_if_present - Functionally identical to dyn_cast, except that a null (or none in the case ...

LLVM_ABI void UpgradeAttributes(AttrBuilder &B)

Upgrade attributes that changed format or kind.

Definition AutoUpgrade.cpp:6382

LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)

This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...

Definition AutoUpgrade.cpp:5401

LLVM_ABI void UpgradeNVVMAnnotations(Module &M)

Convert legacy nvvm.annotations metadata to appropriate function attributes.

Definition AutoUpgrade.cpp:5634

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

LLVM_ABI bool UpgradeModuleFlags(Module &M)

This checks for module flags which should be upgraded.

Definition AutoUpgrade.cpp:5821

std::string utostr(uint64_t X, bool isNeg=false)

constexpr bool isPowerOf2_64(uint64_t Value)

Return true if the argument is a power of two > 0 (64 bit edition.)

void copyModuleAttrToFunctions(Module &M)

Copies module attributes to the functions in the module.

Definition AutoUpgrade.cpp:6104

LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)

Upgrade operand bundles (without knowing about their user instruction).

Definition AutoUpgrade.cpp:6409

LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)

This is an auto-upgrade for bitcast constant expression between pointers with different address space...

Definition AutoUpgrade.cpp:5468

auto dyn_cast_or_null(const Y &Val)

FunctionAddr VTableAddr uintptr_t uintptr_t Version

constexpr bool isPowerOf2_32(uint32_t Value)

Return true if the argument is a power of two > 0.

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)

Upgrade the datalayout string by adding a section for address space pointers.

Definition AutoUpgrade.cpp:6255

bool none_of(R &&Range, UnaryPredicate P)

Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)

This checks for global variables which should be upgraded.

Definition AutoUpgrade.cpp:1767

LLVM_ABI raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

LLVM_ABI bool StripDebugInfo(Module &M)

Strip debug info in the module if it exists.

AtomicOrdering

Atomic ordering for LLVM's memory model.

@ Ref

The access may reference the value stored in memory.

std::string join(IteratorT Begin, IteratorT End, StringRef Separator)

Joins the strings in the range [Begin, End), adding Separator between the elements.

FunctionAddr VTableAddr uintptr_t uintptr_t Data

OperandBundleDefT< Value * > OperandBundleDef

LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)

This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...

Definition AutoUpgrade.cpp:5446

DWARFExpression::Operation Op

@ Dynamic

Denotes mode unknown at compile time.

ArrayRef(const T &OneElt) -> ArrayRef< T >

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

auto find_if(R &&Range, UnaryPredicate P)

Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.

void erase_if(Container &C, UnaryPredicate P)

Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...

LLVM_ABI bool UpgradeDebugInfo(Module &M)

Check the debug info version number, if it is out-dated, drop the debug info.

Definition AutoUpgrade.cpp:5490

LLVM_ABI void UpgradeFunctionAttributes(Function &F)

Correct any IR that is relying on old function attribute behavior.

Definition AutoUpgrade.cpp:6037

@ Default

The result values are uniform if and only if all operands are uniform.

LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)

If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...

Definition AutoUpgrade.cpp:5420

LLVM_ABI void UpgradeARCRuntime(Module &M)

Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...

Definition AutoUpgrade.cpp:5699

LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)

Check a module for errors.

LLVM_ABI void reportFatalUsageError(Error Err)

Report a fatal error that does not indicate a bug in LLVM.

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.

This struct is a compact representation of a valid (non-zero power of two) alignment.

This struct is a compact representation of a valid (power of two) or undefined (0) alignment.