AArch64Arm64ECCallLowering.cpp Source File (original) (raw)

34using namespace llvm;

39#define DEBUG_TYPE "arm64eccalllowering"

41 STATISTIC(Arm64ECCallsLowered, "Number of Arm64EC calls lowered");

48namespace {

50enum ThunkArgTranslation : uint8_t {

51 Direct,

52 Bitcast,

53 PointerIndirection,

54};

56struct ThunkArgInfo {

57 Type *Arm64Ty;

59 ThunkArgTranslation Translation;

60};

62class AArch64Arm64ECCallLowering : public ModulePass {

63public:

64 static char ID;

65 AArch64Arm64ECCallLowering() : ModulePass(ID) {}

67 Function *buildExitThunk(FunctionType *FnTy, AttributeList Attrs);

68 Function *buildEntryThunk(Function *F);

69 void lowerCall(CallBase *CB);

70 Function *buildGuestExitThunk(Function *F);

71 Function *buildPatchableThunk(GlobalAlias *UnmangledAlias,

72 GlobalAlias *MangledAlias);

73 bool processFunction(Function &F, SetVector<GlobalValue *> &DirectCalledFns,

74 DenseMap<GlobalAlias *, GlobalAlias *> &FnsMap);

75 bool runOnModule(Module &M) override;

77private:

78 int cfguard_module_flag = 0;

79 FunctionType *GuardFnType = nullptr;

80 FunctionType *DispatchFnType = nullptr;

81 Constant *GuardFnCFGlobal = nullptr;

82 Constant *GuardFnGlobal = nullptr;

83 Constant *DispatchFnGlobal = nullptr;

90 void getThunkType(FunctionType *FT, AttributeList AttrList,

92 FunctionType *&Arm64Ty, FunctionType *&X64Ty,

94 void getThunkRetType(FunctionType *FT, AttributeList AttrList,

95 raw_ostream &Out, Type *&Arm64RetTy, Type *&X64RetTy,

96 SmallVectorImpl<Type *> &Arm64ArgTypes,

97 SmallVectorImpl<Type *> &X64ArgTypes,

99 bool &HasSretPtr);

100 void getThunkArgTypes(FunctionType *FT, AttributeList AttrList,

102 SmallVectorImpl<Type *> &Arm64ArgTypes,

103 SmallVectorImpl<Type *> &X64ArgTypes,

104 SmallVectorImpl &ArgTranslations,

105 bool HasSretPtr);

106 ThunkArgInfo canonicalizeThunkType(Type *T, Align Alignment, bool Ret,

107 uint64_t ArgSizeBytes, raw_ostream &Out);

108};

109

110}

111

112void AArch64Arm64ECCallLowering::getThunkType(

116 Out << (TT == Arm64ECThunkType::Entry ? "$ientry_thunk$cdecl$"

117 : "$iexit_thunk$cdecl$");

118

119 Type *Arm64RetTy;

120 Type *X64RetTy;

121

124

125

126

127

128 if (TT == Arm64ECThunkType::Exit)

131

132 bool HasSretPtr = false;

133 getThunkRetType(FT, AttrList, Out, Arm64RetTy, X64RetTy, Arm64ArgTypes,

134 X64ArgTypes, ArgTranslations, HasSretPtr);

135

136 getThunkArgTypes(FT, AttrList, TT, Out, Arm64ArgTypes, X64ArgTypes,

137 ArgTranslations, HasSretPtr);

138

139 Arm64Ty = FunctionType::get(Arm64RetTy, Arm64ArgTypes, false);

140

141 X64Ty = FunctionType::get(X64RetTy, X64ArgTypes, false);

142}

143

144void AArch64Arm64ECCallLowering::getThunkArgTypes(

145 FunctionType *FT, AttributeList AttrList, Arm64ECThunkType TT,

146 raw_ostream &Out, SmallVectorImpl<Type *> &Arm64ArgTypes,

147 SmallVectorImpl<Type *> &X64ArgTypes,

148 SmallVectorImpl &ArgTranslations, bool HasSretPtr) {

149

150 Out << "$";

151 if (FT->isVarArg()) {

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171 Out << "varargs";

172

173

174 for (int i = HasSretPtr ? 1 : 0; i < 4; i++) {

177 ArgTranslations.push_back(ThunkArgTranslation::Direct);

178 }

179

180

183 ArgTranslations.push_back(ThunkArgTranslation::Direct);

184

186 if (TT != Arm64ECThunkType::Entry) {

187

188

190 ArgTranslations.push_back(ThunkArgTranslation::Direct);

191 }

192 return;

193 }

194

195 unsigned I = 0;

196 if (HasSretPtr)

197 I++;

198

199 if (I == FT->getNumParams()) {

200 Out << "v";

201 return;

202 }

203

204 for (unsigned E = FT->getNumParams(); I != E; ++I) {

205#if 0

206

207

208 uint64_t ArgSizeBytes = AttrList.getParamArm64ECArgSizeBytes(I);

209 Align ParamAlign = AttrList.getParamAlignment(I).valueOrOne();

210#else

211 uint64_t ArgSizeBytes = 0;

213#endif

214 auto [Arm64Ty, X64Ty, ArgTranslation] =

215 canonicalizeThunkType(FT->getParamType(I), ParamAlign,

216 false, ArgSizeBytes, Out);

217 Arm64ArgTypes.push_back(Arm64Ty);

219 ArgTranslations.push_back(ArgTranslation);

220 }

221}

222

223void AArch64Arm64ECCallLowering::getThunkRetType(

224 FunctionType *FT, AttributeList AttrList, raw_ostream &Out,

225 Type *&Arm64RetTy, Type *&X64RetTy, SmallVectorImpl<Type *> &Arm64ArgTypes,

226 SmallVectorImpl<Type *> &X64ArgTypes,

228 Type *T = FT->getReturnType();

229#if 0

230

231

232 uint64_t ArgSizeBytes = AttrList.getRetArm64ECArgSizeBytes();

233#else

234 int64_t ArgSizeBytes = 0;

235#endif

236 if (T->isVoidTy()) {

237 if (FT->getNumParams()) {

238 Attribute SRetAttr0 = AttrList.getParamAttr(0, Attribute::StructRet);

239 Attribute InRegAttr0 = AttrList.getParamAttr(0, Attribute::InReg);

241 if (FT->getNumParams() > 1) {

242

243

244

245 SRetAttr1 = AttrList.getParamAttr(1, Attribute::StructRet);

246 InRegAttr1 = AttrList.getParamAttr(1, Attribute::InReg);

247 }

250

251

252

253

254

255 Out << "i8";

256 Arm64RetTy = I64Ty;

257 X64RetTy = I64Ty;

258 return;

259 }

260 if (SRetAttr0.isValid()) {

261

262

263

264

265

266

268 Align SRetAlign = AttrList.getParamAlignment(0).valueOrOne();

269 canonicalizeThunkType(SRetType, SRetAlign, true, ArgSizeBytes,

270 Out);

271 Arm64RetTy = VoidTy;

272 X64RetTy = VoidTy;

273 Arm64ArgTypes.push_back(FT->getParamType(0));

274 X64ArgTypes.push_back(FT->getParamType(0));

275 ArgTranslations.push_back(ThunkArgTranslation::Direct);

276 HasSretPtr = true;

277 return;

278 }

279 }

280

281 Out << "v";

282 Arm64RetTy = VoidTy;

283 X64RetTy = VoidTy;

284 return;

285 }

286

288 canonicalizeThunkType(T, Align(), true, ArgSizeBytes, Out);

289 Arm64RetTy = info.Arm64Ty;

290 X64RetTy = info.X64Ty;

292

293

294

296 X64RetTy = VoidTy;

297 }

298}

299

300ThunkArgInfo AArch64Arm64ECCallLowering::canonicalizeThunkType(

301 Type *T, Align Alignment, bool Ret, uint64_t ArgSizeBytes,

302 raw_ostream &Out) {

303

304 auto direct = [](Type *T) {

305 return ThunkArgInfo{T, T, ThunkArgTranslation::Direct};

306 };

307

308 auto bitcast = [this](Type *Arm64Ty, uint64_t SizeInBytes) {

309 return ThunkArgInfo{Arm64Ty,

311 ThunkArgTranslation::Bitcast};

312 };

313

314 auto pointerIndirection = [this](Type *Arm64Ty) {

315 return ThunkArgInfo{Arm64Ty, PtrTy,

316 ThunkArgTranslation::PointerIndirection};

317 };

318

319 if (T->isHalfTy()) {

320

321 Out << "__llvm_h__";

322 return direct(T);

323 }

324

325 if (T->isFloatTy()) {

326 Out << "f";

327 return direct(T);

328 }

329

330 if (T->isDoubleTy()) {

331 Out << "d";

332 return direct(T);

333 }

334

335 if (T->isFloatingPointTy()) {

336 report_fatal_error("Only 16, 32, and 64 bit floating points are supported "

337 "for ARM64EC thunks");

338 }

339

341

343 if (StructTy->getNumElements() == 1)

344 T = StructTy->getElementType(0);

345

346 if (T->isArrayTy()) {

347 Type *ElementTy = T->getArrayElementType();

348 uint64_t ElementCnt = T->getArrayNumElements();

349 uint64_t ElementSizePerBytes = DL.getTypeSizeInBits(ElementTy) / 8;

350 uint64_t TotalSizeBytes = ElementCnt * ElementSizePerBytes;

354

355 Out << "__llvm_H__";

357 Out << "F";

359 Out << "D";

360 Out << TotalSizeBytes;

361 if (Alignment.value() >= 16 && !Ret)

362 Out << "a" << Alignment.value();

363 if (TotalSizeBytes <= 8) {

364

365

366 return bitcast(T, TotalSizeBytes);

367 } else {

368

369 return pointerIndirection(T);

370 }

371 } else if (T->isFloatingPointTy()) {

373 "Only 16, 32, and 64 bit floating points are supported "

374 "for ARM64EC thunks");

375 }

376 }

377

378 if ((T->isIntegerTy() || T->isPointerTy()) && DL.getTypeSizeInBits(T) <= 64) {

379 Out << "i8";

380 return direct(I64Ty);

381 }

382

383 unsigned TypeSize = ArgSizeBytes;

384 if (TypeSize == 0)

385 TypeSize = DL.getTypeSizeInBits(T) / 8;

386 Out << "m";

387 if (TypeSize != 4)

388 Out << TypeSize;

389 if (Alignment.value() >= 16 && !Ret)

390 Out << "a" << Alignment.value();

391

392 if (TypeSize == 1 || TypeSize == 2 || TypeSize == 4 || TypeSize == 8) {

393

394 return bitcast(T, TypeSize);

395 } else {

396

397 return pointerIndirection(T);

398 }

399}

400

401

402

403Function *AArch64Arm64ECCallLowering::buildExitThunk(FunctionType *FT,

404 AttributeList Attrs) {

405 SmallString<256> ExitThunkName;

406 llvm::raw_svector_ostream ExitThunkStream(ExitThunkName);

407 FunctionType *Arm64Ty, *X64Ty;

409 getThunkType(FT, Attrs, Arm64ECThunkType::Exit, ExitThunkStream, Arm64Ty,

410 X64Ty, ArgTranslations);

411 if (Function *F = M->getFunction(ExitThunkName))

412 return F;

413

415 ExitThunkName, M);

416 F->setCallingConv(CallingConv::ARM64EC_Thunk_Native);

417 F->setSection(".wowthk$aa");

419

420 F->addFnAttr("frame-pointer", "all");

421

422

423

424 if (FT->getNumParams()) {

425 auto SRet = Attrs.getParamAttr(0, Attribute::StructRet);

426 auto InReg = Attrs.getParamAttr(0, Attribute::InReg);

427 if (SRet.isValid() && !InReg.isValid())

428 F->addParamAttr(1, SRet);

429 }

430

431

434 Value *CalleePtr =

435 M->getOrInsertGlobal("__os_arm64x_dispatch_call_no_redirect", PtrTy);

436 Value *Callee = IRB.CreateLoad(PtrTy, CalleePtr);

439

440

441 auto X64TyOffset = 1;

442 Args.push_back(F->arg_begin());

443

444 Type *RetTy = Arm64Ty->getReturnType();

445 if (RetTy != X64Ty->getReturnType()) {

446

447

448

449 if (DL.getTypeStoreSize(RetTy) > 8) {

450 Args.push_back(IRB.CreateAlloca(RetTy));

451 X64TyOffset++;

452 }

453 }

454

455 for (auto [Arg, X64ArgType, ArgTranslation] : llvm::zip_equal(

456 make_range(F->arg_begin() + 1, F->arg_end()),

457 make_range(X64Ty->param_begin() + X64TyOffset, X64Ty->param_end()),

458 ArgTranslations)) {

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473 if (ArgTranslation != ThunkArgTranslation::Direct) {

474 Value *Mem = IRB.CreateAlloca(Arg.getType());

475 IRB.CreateStore(&Arg, Mem);

476 if (ArgTranslation == ThunkArgTranslation::Bitcast) {

477 Type *IntTy = IRB.getIntNTy(DL.getTypeStoreSizeInBits(Arg.getType()));

478 Args.push_back(IRB.CreateLoad(IntTy, Mem));

479 } else {

480 assert(ArgTranslation == ThunkArgTranslation::PointerIndirection);

481 Args.push_back(Mem);

482 }

483 } else {

484 Args.push_back(&Arg);

485 }

486 assert(Args.back()->getType() == X64ArgType);

487 }

488

489

490 CallInst *Call = IRB.CreateCall(X64Ty, Callee, Args);

492

494 if (RetTy != X64Ty->getReturnType()) {

495

496

497 if (DL.getTypeStoreSize(RetTy) > 8) {

498 RetVal = IRB.CreateLoad(RetTy, Args[1]);

499 } else {

500 Value *CastAlloca = IRB.CreateAlloca(RetTy);

501 IRB.CreateStore(Call, CastAlloca);

502 RetVal = IRB.CreateLoad(RetTy, CastAlloca);

503 }

504 }

505

507 IRB.CreateRetVoid();

508 else

509 IRB.CreateRet(RetVal);

510 return F;

511}

512

513

514

515Function *AArch64Arm64ECCallLowering::buildEntryThunk(Function *F) {

516 SmallString<256> EntryThunkName;

517 llvm::raw_svector_ostream EntryThunkStream(EntryThunkName);

518 FunctionType *Arm64Ty, *X64Ty;

520 getThunkType(F->getFunctionType(), F->getAttributes(),

521 Arm64ECThunkType::Entry, EntryThunkStream, Arm64Ty, X64Ty,

522 ArgTranslations);

523 if (Function *F = M->getFunction(EntryThunkName))

524 return F;

525

527 EntryThunkName, M);

528 Thunk->setCallingConv(CallingConv::ARM64EC_Thunk_X64);

529 Thunk->setSection(".wowthk$aa");

531

532 Thunk->addFnAttr("frame-pointer", "all");

533

536

537 Type *RetTy = Arm64Ty->getReturnType();

538 Type *X64RetType = X64Ty->getReturnType();

539

540 bool TransformDirectToSRet = X64RetType->isVoidTy() && !RetTy->isVoidTy();

541 unsigned ThunkArgOffset = TransformDirectToSRet ? 2 : 1;

542 unsigned PassthroughArgSize =

543 (F->isVarArg() ? 5 : Thunk->arg_size()) - ThunkArgOffset;

544 assert(ArgTranslations.size() == (F->isVarArg() ? 5 : PassthroughArgSize));

545

546

548 for (unsigned i = 0; i != PassthroughArgSize; ++i) {

549 Value *Arg = Thunk->getArg(i + ThunkArgOffset);

550 Type *ArgTy = Arm64Ty->getParamType(i);

551 ThunkArgTranslation ArgTranslation = ArgTranslations[i];

552 if (ArgTranslation != ThunkArgTranslation::Direct) {

553

554 if (ArgTranslation == ThunkArgTranslation::Bitcast) {

555 Value *CastAlloca = IRB.CreateAlloca(ArgTy);

556 IRB.CreateStore(Arg, CastAlloca);

557 Arg = IRB.CreateLoad(ArgTy, CastAlloca);

558 } else {

559 assert(ArgTranslation == ThunkArgTranslation::PointerIndirection);

560 Arg = IRB.CreateLoad(ArgTy, Arg);

561 }

562 }

564 Args.push_back(Arg);

565 }

566

567 if (F->isVarArg()) {

568

569

570

571

572

573

574 Thunk->addParamAttr(5, Attribute::InReg);

576 Arg = IRB.CreatePtrAdd(Arg, IRB.getInt64(0x20));

577 Args.push_back(Arg);

578

579

580 Args.push_back(IRB.getInt64(0));

581 }

582

583

585 CallInst *Call = IRB.CreateCall(Arm64Ty, Callee, Args);

586

587 auto SRetAttr = F->getAttributes().getParamAttr(0, Attribute::StructRet);

588 auto InRegAttr = F->getAttributes().getParamAttr(0, Attribute::InReg);

589 if (SRetAttr.isValid() && !InRegAttr.isValid()) {

590 Thunk->addParamAttr(1, SRetAttr);

592 }

593

595 if (TransformDirectToSRet) {

596 IRB.CreateStore(RetVal, Thunk->getArg(1));

597 } else if (X64RetType != RetTy) {

598 Value *CastAlloca = IRB.CreateAlloca(X64RetType);

599 IRB.CreateStore(Call, CastAlloca);

600 RetVal = IRB.CreateLoad(X64RetType, CastAlloca);

601 }

602

603

604

605

606

608 IRB.CreateRetVoid();

609 else

610 IRB.CreateRet(RetVal);

611

613}

614

622

623

624

625

626

627Function *AArch64Arm64ECCallLowering::buildGuestExitThunk(Function *F) {

628 llvm::raw_null_ostream NullThunkName;

629 FunctionType *Arm64Ty, *X64Ty;

631 getThunkType(F->getFunctionType(), F->getAttributes(),

632 Arm64ECThunkType::GuestExit, NullThunkName, Arm64Ty, X64Ty,

633 ArgTranslations);

635 assert(MangledName && "Can't guest exit to function that's already native");

636 std::string ThunkName = *MangledName;

637 if (ThunkName[0] == '?' && ThunkName.find("@") != std:🧵:npos) {

638 ThunkName.insert(ThunkName.find("@"), "$exit_thunk");

639 } else {

640 ThunkName.append("$exit_thunk");

641 }

645 GuestExit->setSection(".wowthk$aa");

647 "arm64ec_unmangled_name",

651 "arm64ec_ecmangled_name",

657

658

659

660

661 LoadInst *GuardCheckLoad = B.CreateLoad(PtrTy, GuardFnGlobal);

662 Function *Thunk = buildExitThunk(F->getFunctionType(), F->getAttributes());

663 CallInst *GuardCheck = B.CreateCall(

664 GuardFnType, GuardCheckLoad, {F, Thunk});

665 Value *GuardCheckDest = B.CreateExtractValue(GuardCheck, 0);

666 Value *GuardFinalDest = B.CreateExtractValue(GuardCheck, 1);

667

668

669 GuardCheck->setCallingConv(CallingConv::CFGuard_Check);

670

673 CallInst *Call = B.CreateCall(Arm64Ty, GuardCheckDest, Args, OB);

675

677 B.CreateRetVoid();

678 else

679 B.CreateRet(Call);

680

681 auto SRetAttr = F->getAttributes().getParamAttr(0, Attribute::StructRet);

682 auto InRegAttr = F->getAttributes().getParamAttr(0, Attribute::InReg);

683 if (SRetAttr.isValid() && !InRegAttr.isValid()) {

684 GuestExit->addParamAttr(0, SRetAttr);

686 }

687

689}

690

692AArch64Arm64ECCallLowering::buildPatchableThunk(GlobalAlias *UnmangledAlias,

693 GlobalAlias *MangledAlias) {

694 llvm::raw_null_ostream NullThunkName;

695 FunctionType *Arm64Ty, *X64Ty;

698 getThunkType(F->getFunctionType(), F->getAttributes(),

699 Arm64ECThunkType::GuestExit, NullThunkName, Arm64Ty, X64Ty,

700 ArgTranslations);

701 std::string ThunkName(MangledAlias->getName());

702 if (ThunkName[0] == '?' && ThunkName.find("@") != std:🧵:npos) {

703 ThunkName.insert(ThunkName.find("@"), "$hybpatch_thunk");

704 } else {

705 ThunkName.append("$hybpatch_thunk");

706 }

707

711 GuestExit->setSection(".wowthk$aa");

714

715

716 LoadInst *DispatchLoad = B.CreateLoad(PtrTy, DispatchFnGlobal);

717

718

720 buildExitThunk(F->getFunctionType(), F->getAttributes());

721 CallInst *Dispatch =

722 B.CreateCall(DispatchFnType, DispatchLoad,

723 {UnmangledAlias, ExitThunk, UnmangledAlias->getAliasee()});

724

725

726 Dispatch->setCallingConv(CallingConv::CFGuard_Check);

727

729 CallInst *Call = B.CreateCall(Arm64Ty, Dispatch, Args);

731

733 B.CreateRetVoid();

734 else

735 B.CreateRet(Call);

736

737 auto SRetAttr = F->getAttributes().getParamAttr(0, Attribute::StructRet);

738 auto InRegAttr = F->getAttributes().getParamAttr(0, Attribute::InReg);

739 if (SRetAttr.isValid() && !InRegAttr.isValid()) {

740 GuestExit->addParamAttr(0, SRetAttr);

742 }

743

746}

747

748

749void AArch64Arm64ECCallLowering::lowerCall(CallBase *CB) {

752

753

754

758

759

761 if (cfguard_module_flag == 2 && !CB->hasFnAttr("guard_nocf"))

762 GuardFn = GuardFnCFGlobal;

763 else

764 GuardFn = GuardFnGlobal;

765 LoadInst *GuardCheckLoad = B.CreateLoad(PtrTy, GuardFn);

766

767

768

770 CallInst *GuardCheck =

771 B.CreateCall(GuardFnType, GuardCheckLoad, {CalledOperand, Thunk},

772 Bundles);

773 Value *GuardCheckDest = B.CreateExtractValue(GuardCheck, 0);

774 Value *GuardFinalDest = B.CreateExtractValue(GuardCheck, 1);

775

776

777 GuardCheck->setCallingConv(CallingConv::CFGuard_Check);

778

779

780

785 NewCall->copyMetadata(*CB);

788}

789

790bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) {

792 return false;

793

794 M = &Mod;

795

796

797 if (auto *MD =

799 cfguard_module_flag = MD->getZExtValue();

800

801 PtrTy = PointerType::getUnqual(M->getContext());

802 I64Ty = Type::getInt64Ty(M->getContext());

803 VoidTy = Type::getVoidTy(M->getContext());

804

805 GuardFnType =

806 FunctionType::get(StructType::get(PtrTy, PtrTy), {PtrTy, PtrTy}, false);

807 DispatchFnType = FunctionType::get(PtrTy, {PtrTy, PtrTy, PtrTy}, false);

808 GuardFnCFGlobal = M->getOrInsertGlobal("__os_arm64x_check_icall_cfg", PtrTy);

809 GuardFnGlobal = M->getOrInsertGlobal("__os_arm64x_check_icall", PtrTy);

810 DispatchFnGlobal = M->getOrInsertGlobal("__os_arm64x_dispatch_call", PtrTy);

811

812

813

814

815

816 for (GlobalAlias &A : Mod.aliases()) {

818 if ()

819 continue;

820 if (std::optionalstd::string MangledName =

822 F->addMetadata("arm64ec_unmangled_name",

825 A.setName(MangledName.value());

826 }

827 }

828

829 DenseMap<GlobalAlias *, GlobalAlias *> FnsMap;

830 SetVector<GlobalAlias *> PatchableFns;

831

832 for (Function &F : Mod) {

833 if (F.hasPersonalityFn()) {

834 GlobalValue *PersFn =

837 if (std::optionalstd::string MangledName =

839 PersFn->setName(MangledName.value());

840 }

841 }

842 }

843

844 if (.hasFnAttribute(Attribute::HybridPatchable) ||

845 F.isDeclarationForLinker() || F.hasLocalLinkage() ||

847 continue;

848

849

850

851 if (std::optionalstd::string MangledName =

853 std::string OrigName(F.getName());

855

856

857

858

859

860

861

862 auto *A =

865 MangledName.value(), &F);

866 F.replaceUsesWithIf(AM,

868 F.replaceAllUsesWith(A);

869 F.setMetadata("arm64ec_exp_name",

872 "EXP+" + MangledName.value())));

873 A->setAliasee(&F);

874 AM->setAliasee(&F);

875

876 if (F.hasDLLExportStorageClass()) {

879 }

880

881 FnsMap[A] = AM;

883 }

884 }

885

886 SetVector<GlobalValue *> DirectCalledFns;

887 for (Function &F : Mod)

888 if (.isDeclarationForLinker() &&

889 F.getCallingConv() != CallingConv::ARM64EC_Thunk_Native &&

890 F.getCallingConv() != CallingConv::ARM64EC_Thunk_X64)

892

893 struct ThunkInfo {

897 };

899 for (Function &F : Mod) {

900 if (.isDeclarationForLinker() &&

901 (.hasLocalLinkage() || F.hasAddressTaken()) &&

902 F.getCallingConv() != CallingConv::ARM64EC_Thunk_Native &&

903 F.getCallingConv() != CallingConv::ARM64EC_Thunk_X64) {

904 if (.hasComdat())

905 F.setComdat(Mod.getOrInsertComdat(F.getName()));

907 {&F, buildEntryThunk(&F), Arm64ECThunkType::Entry});

908 }

909 }

910 for (GlobalValue *O : DirectCalledFns) {

914 {O, buildExitThunk(F->getFunctionType(), F->getAttributes()),

915 Arm64ECThunkType::Exit});

916 if (!GA && ->hasDLLImportStorageClass())

918 {buildGuestExitThunk(F), F, Arm64ECThunkType::GuestExit});

919 }

920 for (GlobalAlias *A : PatchableFns) {

922 ThunkMapping.push_back({Thunk, A, Arm64ECThunkType::GuestExit});

923 }

924

925 if (!ThunkMapping.empty()) {

927 for (ThunkInfo &Thunk : ThunkMapping) {

930 ConstantInt::get(M->getContext(), APInt(32, uint8_t(Thunk.Kind)))}));

931 }

934 ThunkMappingArrayElems.size()),

935 ThunkMappingArrayElems);

936 new GlobalVariable(Mod, ThunkMappingArray->getType(), false,

938 "llvm.arm64ec.symbolmap");

939 }

940

941 return true;

942}

943

944bool AArch64Arm64ECCallLowering::processFunction(

945 Function &F, SetVector<GlobalValue *> &DirectCalledFns,

946 DenseMap<GlobalAlias *, GlobalAlias *> &FnsMap) {

948

949

950

951

952

953

954

955

956 if (.hasLocalLinkage() || F.hasAddressTaken()) {

957 if (std::optionalstd::string MangledName =

959 F.addMetadata("arm64ec_unmangled_name",

962 if (F.hasComdat() && F.getComdat()->getName() == F.getName()) {

963 Comdat *MangledComdat = M->getOrInsertComdat(MangledName.value());

965 to_vector(F.getComdat()->getUsers());

966 for (GlobalObject *User : ComdatUsers)

967 User->setComdat(MangledComdat);

968 }

969 F.setName(MangledName.value());

970 }

971 }

972

973

974

975

976

977 for (BasicBlock &BB : F) {

978 for (Instruction &I : BB) {

980 if (!CB || CB->getCallingConv() == CallingConv::ARM64EC_Thunk_X64 ||

982 continue;

983

984

985

986

987

988

991 F->isIntrinsic() || ->isDeclarationForLinker())

992 continue;

993

994 DirectCalledFns.insert(F);

995 continue;

996 }

997

998

1000 auto I = FnsMap.find(A);

1001 if (I != FnsMap.end()) {

1003 DirectCalledFns.insert(I->first);

1004 continue;

1005 }

1006 }

1007

1009 ++Arm64ECCallsLowered;

1010 }

1011 }

1012

1013 if (IndirectCalls.empty())

1014 return false;

1015

1016 for (CallBase *CB : IndirectCalls)

1017 lowerCall(CB);

1018

1019 return true;

1020}

1021

1022char AArch64Arm64ECCallLowering::ID = 0;

1024 "AArch64Arm64ECCallLowering", false, false)

1025

1027 return new AArch64Arm64ECCallLowering;

1028}

static cl::opt< bool > LowerDirectToIndirect("arm64ec-lower-direct-to-indirect", cl::Hidden, cl::init(true))

static cl::opt< bool > GenerateThunks("arm64ec-generate-thunks", cl::Hidden, cl::init(true))

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

Module.h This file contains the declarations for the Module class.

Machine Check Debug Module

static bool processFunction(Function &F, NVPTXTargetMachine &TM)

if(auto Err=PB.parsePassPipeline(MPM, Passes)) return wrap(std MPM run * Mod

#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)

This file implements a set that has insertion order iteration characteristics.

This file defines the SmallString class.

This file defines the SmallVector class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

static SymbolRef::Type getType(const Symbol *Sym)

static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)

This static method is the primary way to construct an ArrayType.

bool isValid() const

Return true if the attribute is any kind of attribute.

LLVM_ABI Type * getValueAsType() const

Return the attribute's value as a Type.

static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)

Creates a new BasicBlock.

bool isInlineAsm() const

Check if this call is an inline asm statement.

void setCallingConv(CallingConv::ID CC)

std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const

Return an operand bundle by name, if present.

Function * getCalledFunction() const

Returns the function called, or null if this is an indirect function invocation or the function signa...

bool hasFnAttr(Attribute::AttrKind Kind) const

Determine whether this call has the given attribute.

CallingConv::ID getCallingConv() const

static LLVM_ABI CallBase * addOperandBundle(CallBase *CB, uint32_t ID, OperandBundleDef OB, InsertPosition InsertPt=nullptr)

Create a clone of CB with operand bundle OB added.

Value * getCalledOperand() const

FunctionType * getFunctionType() const

void setCalledOperand(Value *V)

AttributeList getAttributes() const

Return the attributes for this call.

void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)

Adds the attribute to the indicated argument.

void setTailCallKind(TailCallKind TCK)

static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)

static Constant * getAnon(ArrayRef< Constant * > V, bool Packed=false)

Return an anonymous struct that has the specified elements.

iterator find(const_arg_type_t< KeyT > Val)

static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)

LLVM_ABI void setAliasee(Constant *Aliasee)

These methods retrieve and set alias target.

const Constant * getAliasee() const

static LLVM_ABI GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)

If a parent module is specified, the alias is automatically inserted into the end of the specified mo...

@ DLLExportStorageClass

Function to be accessible from DLL.

@ WeakODRLinkage

Same, but only replaced by something equivalent.

@ ExternalLinkage

Externally visible function.

@ LinkOnceODRLinkage

Same, but only replaced by something equivalent.

Type * getValueType() const

LLVM_ABI InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)

ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...

LLVMContext & getContext() const

Get the global data context.

Function * getFunction(StringRef Name) const

Look up the specified function in the module symbol table.

Comdat * getOrInsertComdat(StringRef Name)

Return the Comdat in the module with the specified name.

const DataLayout & getDataLayout() const

Get the data layout for the module's target platform.

GlobalVariable * getOrInsertGlobal(StringRef Name, Type *Ty, function_ref< GlobalVariable *()> CreateGlobalCallback)

Look up the specified global in the module symbol table.

Metadata * getModuleFlag(StringRef Key) const

Return the corresponding value if Key appears in module flags, otherwise return null.

A container for an operand bundle being viewed as a set of values rather than a set of uses.

bool insert(const value_type &X)

Insert a new element into the SetVector.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)

This static method is the primary way to create a literal StructType.

bool isPointerTy() const

True if this is an instance of PointerType.

bool isFloatTy() const

Return true if this is 'float', a 32-bit IEEE fp type.

bool isHalfTy() const

Return true if this is 'half', a 16-bit IEEE fp type.

bool isDoubleTy() const

Return true if this is 'double', a 64-bit IEEE fp type.

bool isFunctionTy() const

True if this is an instance of FunctionType.

static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)

bool isVoidTy() const

Return true if this is 'void'.

Type * getType() const

All values are typed, get the type of this value.

LLVM_ABI void setName(const Twine &Name)

Change the name of the value.

LLVM_ABI void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

self_iterator getIterator()

This class implements an extremely fast bulk output stream that can only output to a stream.

constexpr char Align[]

Key for Kernel::Arg::Metadata::mAlign.

constexpr char Args[]

Key for Kernel::Metadata::mArgs.

constexpr char Attrs[]

Key for Kernel::Metadata::mAttrs.

@ BasicBlock

Various leaf nodes.

@ OB

OB - OneByte - Set if this instruction has a one byte opcode.

initializer< Ty > init(const Ty &Val)

std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract_or_null(Y &&MD)

Extract a Value from Metadata, allowing null.

@ User

could "use" a pointer

This is an optimization pass for GlobalISel generic memory operations.

FunctionAddr VTableAddr Value

LLVM_ABI std::optional< std::string > getArm64ECMangledFunctionName(StringRef Name)

Returns the ARM64EC mangled function name unless the input is already mangled.

detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)

zip iterator that assumes that all iteratees have the same length.

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

iterator_range< T > make_range(T x, T y)

Convenience function for iterating over sub-ranges.

auto dyn_cast_or_null(const Y &Val)

ModulePass * createAArch64Arm64ECCallLoweringPass()

LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)

SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)

Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

constexpr std::string_view HybridPatchableTargetSuffix

IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >

OperandBundleDefT< Value * > OperandBundleDef

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

iterator_range< pointer_iterator< WrappedIteratorT > > make_pointer_range(RangeT &&Range)

constexpr uint64_t value() const

This is a hole in the type system and should not be abused.