ELFNixPlatform.cpp Source File (original) (raw)

19#include

21#define DEBUG_TYPE "orc"

23using namespace llvm;

27namespace {

29template <typename SPSSerializer, typename... ArgTs>

31getArgDataBufferType(const ArgTs &...Args) {

33 ArgData.resize(SPSSerializer::size(Args...));

35 ArgData.size());

36 if (SPSSerializer::serialize(OB, Args...))

37 return ArgData;

38 return {};

39}

41std::unique_ptrjitlink::LinkGraph createPlatformGraph(ELFNixPlatform &MOP,

42 std::string Name) {

44 return std::make_uniquejitlink::LinkGraph(

45 std::move(Name), ES.getSymbolStringPool(), ES.getTargetTriple(),

47}

50class ELFNixPlatformCompleteBootstrapMaterializationUnit

52public:

53 ELFNixPlatformCompleteBootstrapMaterializationUnit(

61 MOP(MOP), PlatformJDName(PlatformJDName),

62 CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),

63 DeferredAAsMap(std::move(DeferredAAs)),

64 ELFNixHeaderAddr(ELFNixHeaderAddr),

65 PlatformBootstrap(PlatformBootstrap),

66 PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),

67 DeregisterJITDylib(DeregisterJITDylib) {}

70 return "ELFNixPlatformCompleteBootstrap";

71 }

73 void materialize(std::unique_ptr R) override {

75 auto G = createPlatformGraph(MOP, "");

76 auto &PlaceholderSection =

78 auto &PlaceholderBlock =

79 G->createZeroFillBlock(PlaceholderSection, 1, ExecutorAddr(), 1, 0);

80 G->addDefinedSymbol(PlaceholderBlock, 0, *CompleteBootstrapSymbol, 1,

81 Linkage::Strong, Scope::Hidden, false, true);

84 G->allocActions().push_back(

86 PlatformBootstrap, ELFNixHeaderAddr)),

91 G->allocActions().push_back(

94 RegisterJITDylib, PlatformJDName, ELFNixHeaderAddr)),

96 DeregisterJITDylib, ELFNixHeaderAddr))});

99 for (auto &[Fn, CallDatas] : DeferredAAsMap) {

100 for (auto &CallData : CallDatas) {

101 G->allocActions().push_back(

104 }

105 }

106

108 }

109

111

112private:

122};

123

125public:

129 createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),

130 ENP(ENP) {}

131

133

134 void materialize(std::unique_ptr R) override {

135

137

138 jitlink::Edge::Kind EdgeKind;

139

140 switch (ES.getTargetTriple().getArch()) {

143 break;

146 break;

149 break;

152 break;

155 break;

156 default:

158 }

159

160

161 auto G = std::make_uniquejitlink::LinkGraph(

162 "", ES.getSymbolStringPool(), ES.getTargetTriple(),

164 auto &DSOHandleSection =

165 G->createSection(".data.__dso_handle", MemProt::Read);

166 auto &DSOHandleBlock = G->createContentBlock(

167 DSOHandleSection, getDSOHandleContent(G->getPointerSize()),

169 auto &DSOHandleSymbol = G->addDefinedSymbol(

170 DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),

172 DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);

173

175 }

176

178

179private:

186 DSOHandleSymbol);

187 }

188

189 ArrayRef getDSOHandleContent(size_t PointerSize) {

190 static const char Content[8] = {0};

191 assert(PointerSize <= sizeof Content);

192 return {Content, PointerSize};

193 }

194

196};

197

198}

199

200namespace llvm {

201namespace orc {

202

203Expected<std::unique_ptr>

206 std::unique_ptr OrcRuntime,

207 std::optional RuntimeAliases) {

208

209 auto &ES = ObjLinkingLayer.getExecutionSession();

210

211

212 if (!supportedTarget(ES.getTargetTriple()))

214 ES.getTargetTriple().str(),

216

217 auto &EPC = ES.getExecutorProcessControl();

218

219

220 if (!RuntimeAliases) {

222 if (!StandardRuntimeAliases)

223 return StandardRuntimeAliases.takeError();

224 RuntimeAliases = std::move(*StandardRuntimeAliases);

225 }

226

227

228 if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))

229 return std::move(Err);

230

231

232 if (auto Err = PlatformJD.define(

234 {EPC.getJITDispatchInfo().JITDispatchFunction,

236 {ES.intern("__orc_rt_jit_dispatch_ctx"),

237 {EPC.getJITDispatchInfo().JITDispatchContext,

239 return std::move(Err);

240

241

243 auto P = std::unique_ptr(new ELFNixPlatform(

244 ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));

245 if (Err)

246 return std::move(Err);

247 return std::move(P);

248}

249

252 JITDylib &PlatformJD, const char *OrcRuntimePath,

253 std::optional RuntimeAliases) {

254

255

256 auto OrcRuntimeArchiveGenerator =

258 if (!OrcRuntimeArchiveGenerator)

259 return OrcRuntimeArchiveGenerator.takeError();

260

261 return Create(ObjLinkingLayer, PlatformJD,

262 std::move(*OrcRuntimeArchiveGenerator),

263 std::move(RuntimeAliases));

264}

265

267 if (auto Err = JD.define(std::make_unique(

268 *this, DSOHandleSymbol)))

269 return Err;

270

271 return ES.lookup({&JD}, DSOHandleSymbol).takeError();

272}

273

275 std::lock_guardstd::mutex Lock(PlatformMutex);

276 auto I = JITDylibToHandleAddr.find(&JD);

277 if (I != JITDylibToHandleAddr.end()) {

278 assert(HandleAddrToJITDylib.count(I->second) &&

279 "HandleAddrToJITDylib missing entry");

280 HandleAddrToJITDylib.erase(I->second);

281 JITDylibToHandleAddr.erase(I);

282 }

284}

285

288

291 if (!InitSym)

293

294 RegisteredInitSymbols[&JD].add(InitSym,

297 dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym

298 << " for MU " << MU.getName() << "\n";

299 });

301}

302

306

308 ArrayRef<std::pair<const char *, const char *>> AL) {

309 for (auto &KV : AL) {

310 auto AliasName = ES.intern(KV.first);

311 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");

312 Aliases[std::move(AliasName)] = {ES.intern(KV.second),

314 }

315}

316

326

329 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {

330 {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},

331 {"atexit", "__orc_rt_elfnix_atexit"}};

332

334}

335

338 static const std::pair<const char *, const char *>

339 StandardRuntimeUtilityAliases[] = {

340 {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},

341 {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},

342 {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},

343 {"__orc_rt_jit_dlupdate", "__orc_rt_elfnix_jit_dlupdate"},

344 {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},

345 {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},

346 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};

347

349 StandardRuntimeUtilityAliases);

350}

351

354 static const std::pair<const char *, const char *>

355 StandardLazyCompilationAliases[] = {

356 {"__orc_rt_reenter", "__orc_rt_sysv_reenter"}};

357

359 StandardLazyCompilationAliases);

360}

361

362bool ELFNixPlatform::supportedTarget(const Triple &TT) {

363 switch (TT.getArch()) {

366

367

370 return true;

371 default:

372 return false;

373 }

374}

375

376ELFNixPlatform::ELFNixPlatform(

378 std::unique_ptr OrcRuntimeGenerator, Error &Err)

379 : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),

380 ObjLinkingLayer(ObjLinkingLayer),

381 DSOHandleSymbol(ES.intern("__dso_handle")) {

383 ObjLinkingLayer.addPlugin(std::make_unique(*this));

384

385 PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));

386

387 BootstrapInfo BI;

388 Bootstrap = &BI;

389

390

391

393 Err = std::move(E2);

394 return;

395 }

396

397

398

399 if ((Err = ES.lookup(

401 SymbolLookupSet(

402 {PlatformBootstrap.Name, PlatformShutdown.Name,

403 RegisterJITDylib.Name, DeregisterJITDylib.Name,

404 RegisterInitSections.Name, DeregisterInitSections.Name,

405 RegisterObjectSections.Name,

406 DeregisterObjectSections.Name, CreatePThreadKey.Name}))

407 .takeError()))

408 return;

409

410

411 {

412 std::unique_lockstd::mutex Lock(BI.Mutex);

413 BI.CV.wait(Lock, [&]() { return BI.ActiveGraphs == 0; });

414 Bootstrap = nullptr;

415 }

416

417

418 auto BootstrapCompleteSymbol =

419 ES.intern("__orc_rt_elfnix_complete_bootstrap");

420 if ((Err = PlatformJD.define(

421 std::make_unique(

422 *this, PlatformJD.getName(), BootstrapCompleteSymbol,

423 std::move(BI.DeferredRTFnMap), BI.ELFNixHeaderAddr,

424 PlatformBootstrap.Addr, PlatformShutdown.Addr,

425 RegisterJITDylib.Addr, DeregisterJITDylib.Addr))))

426 return;

429 std::move(BootstrapCompleteSymbol))

430 .takeError()))

431 return;

432

433

434 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {

435 Err = std::move(E2);

436 return;

437 }

438}

439

440Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {

442

443 using RecordInitializersSPSSig =

444 SPSExpected(SPSExecutorAddr);

445 WFs[ES.intern("__orc_rt_elfnix_push_initializers_tag")] =

446 ES.wrapAsyncWithSPS(

447 this, &ELFNixPlatform::rt_recordInitializers);

448

449 using LookupSymbolSPSSig =

450 SPSExpected(SPSExecutorAddr, SPSString);

451 WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =

452 ES.wrapAsyncWithSPS(this,

453 &ELFNixPlatform::rt_lookupSymbol);

454

455 return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));

456}

457

458void ELFNixPlatform::pushInitializersLoop(

459 PushInitializersSendResultFn SendResult, JITDylibSP JD) {

460 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;

461 DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;

463

464 ES.runSessionLocked([&]() {

465 while (!Worklist.empty()) {

466

467

468 auto DepJD = Worklist.back();

469 Worklist.pop_back();

470

471

473 if (!Inserted)

474 continue;

475

476

477 auto &DM = It->second;

479 for (auto &KV : O) {

480 if (KV.first == DepJD)

481 continue;

482 DM.push_back(KV.first);

483 Worklist.push_back(KV.first);

484 }

485 });

486

487

488 auto RISItr = RegisteredInitSymbols.find(DepJD);

489 if (RISItr != RegisteredInitSymbols.end()) {

490 NewInitSymbols[DepJD] = std::move(RISItr->second);

491 RegisteredInitSymbols.erase(RISItr);

492 }

493 }

494 });

495

496

497

498 if (NewInitSymbols.empty()) {

499

500

501

502

503

504 DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;

506 {

507 std::lock_guardstd::mutex Lock(PlatformMutex);

508 for (auto &KV : JDDepMap) {

509 auto I = JITDylibToHandleAddr.find(KV.first);

510 if (I != JITDylibToHandleAddr.end())

511 HeaderAddrs[KV.first] = I->second;

512 }

513 }

514

515

517 DIM.reserve(JDDepMap.size());

518 for (auto &KV : JDDepMap) {

519 auto HI = HeaderAddrs.find(KV.first);

520

521 if (HI == HeaderAddrs.end())

522 continue;

523 auto H = HI->second;

525 for (auto &Dep : KV.second) {

526 auto HJ = HeaderAddrs.find(Dep);

527 if (HJ != HeaderAddrs.end())

528 DepInfo.push_back(HJ->second);

529 }

530 DIM.push_back(std::make_pair(H, std::move(DepInfo)));

531 }

532 SendResult(DIM);

533 return;

534 }

535

536

538 [this, SendResult = std::move(SendResult), JD](Error Err) mutable {

539 if (Err)

540 SendResult(std::move(Err));

541 else

542 pushInitializersLoop(std::move(SendResult), JD);

543 },

544 ES, std::move(NewInitSymbols));

545}

546

547void ELFNixPlatform::rt_recordInitializers(

548 PushInitializersSendResultFn SendResult, ExecutorAddr JDHeaderAddr) {

550 {

551 std::lock_guardstd::mutex Lock(PlatformMutex);

552 auto I = HandleAddrToJITDylib.find(JDHeaderAddr);

553 if (I != HandleAddrToJITDylib.end())

554 JD = I->second;

555 }

556

558 dbgs() << "ELFNixPlatform::rt_recordInitializers(" << JDHeaderAddr << ") ";

559 if (JD)

560 dbgs() << "pushing initializers for " << JD->getName() << "\n";

561 else

562 dbgs() << "No JITDylib for header address.\n";

563 });

564

565 if (!JD) {

567 formatv("{0:x}", JDHeaderAddr),

569 return;

570 }

571

572 pushInitializersLoop(std::move(SendResult), JD);

573}

574

575void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,

577 StringRef SymbolName) {

579 dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";

580 });

581

582 JITDylib *JD = nullptr;

583

584 {

585 std::lock_guardstd::mutex Lock(PlatformMutex);

586 auto I = HandleAddrToJITDylib.find(Handle);

587 if (I != HandleAddrToJITDylib.end())

588 JD = I->second;

589 }

590

591 if (!JD) {

592 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");

594 formatv("{0:x}", Handle),

596 return;

597 }

598

599

600 class RtLookupNotifyComplete {

601 public:

602 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)

603 : SendResult(std::move(SendResult)) {}

604 void operator()(Expected Result) {

606 assert(Result->size() == 1 && "Unexpected result map count");

607 SendResult(Result->begin()->second.getAddress());

608 } else {

609 SendResult(Result.takeError());

610 }

611 }

612

613 private:

614 SendSymbolAddressFn SendResult;

615 };

616

617 ES.lookup(

621}

622

623Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineStart(

624 jitlink::LinkGraph &G) {

625

626 std::lock_guardstd::mutex Lock(MP.Bootstrap.load()->Mutex);

627 ++MP.Bootstrap.load()->ActiveGraphs;

629}

630

631Error ELFNixPlatform::ELFNixPlatformPlugin::

632 bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {

633

634 std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {

635 {*MP.DSOHandleSymbol, &MP.Bootstrap.load()->ELFNixHeaderAddr},

636 {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},

637 {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},

638 {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},

639 {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},

640 {*MP.RegisterObjectSections.Name, &MP.RegisterObjectSections.Addr},

641 {*MP.DeregisterObjectSections.Name, &MP.DeregisterObjectSections.Addr},

642 {*MP.RegisterInitSections.Name, &MP.RegisterInitSections.Addr},

643 {*MP.DeregisterInitSections.Name, &MP.DeregisterInitSections.Addr},

644 {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};

645

646 bool RegisterELFNixHeader = false;

647

648 for (auto *Sym : G.defined_symbols()) {

649 for (auto &RTSym : RuntimeSymbols) {

650 if (Sym->hasName() && *Sym->getName() == RTSym.first) {

651 if (*RTSym.second)

653 "Duplicate " + RTSym.first +

654 " detected during ELFNixPlatform bootstrap",

656

657 if (*Sym->getName() == *MP.DSOHandleSymbol)

658 RegisterELFNixHeader = true;

659

660 *RTSym.second = Sym->getAddress();

661 }

662 }

663 }

664

665 if (RegisterELFNixHeader) {

666

667

668 std::lock_guardstd::mutex Lock(MP.PlatformMutex);

669 MP.JITDylibToHandleAddr[&MP.PlatformJD] =

670 MP.Bootstrap.load()->ELFNixHeaderAddr;

671 MP.HandleAddrToJITDylib[MP.Bootstrap.load()->ELFNixHeaderAddr] =

672 &MP.PlatformJD;

673 }

674

676}

677

678Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineEnd(

679 jitlink::LinkGraph &G) {

680 std::lock_guardstd::mutex Lock(MP.Bootstrap.load()->Mutex);

681 assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");

682 --MP.Bootstrap.load()->ActiveGraphs;

683

684

685 if (MP.Bootstrap.load()->ActiveGraphs == 0)

686 MP.Bootstrap.load()->CV.notify_all();

688}

689

690Error ELFNixPlatform::registerPerObjectSections(

691 jitlink::LinkGraph &G, const ELFPerObjectSectionsToRegister &POSR,

692 bool IsBootstrapping) {

693 using SPSRegisterPerObjSectionsArgs =

694 SPSArgList;

695

697 Bootstrap.load()->addArgumentsToRTFnMap(

698 &RegisterObjectSections, &DeregisterObjectSections,

699 getArgDataBufferType(POSR),

700 getArgDataBufferType(POSR));

702 }

703

704 G.allocActions().push_back(

706 RegisterObjectSections.Addr, POSR)),

708 DeregisterObjectSections.Addr, POSR))});

709

711}

712

713Expected<uint64_t> ELFNixPlatform::createPThreadKey() {

714 if (!CreatePThreadKey.Addr)

716 "Attempting to create pthread key in target, but runtime support has "

717 "not been loaded yet",

719

720 Expected<uint64_t> Result(0);

721 if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(

722 CreatePThreadKey.Addr, Result))

723 return std::move(Err);

725}

726

727void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(

728 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,

729 jitlink::PassConfiguration &Config) {

730 using namespace jitlink;

731

732 bool InBootstrapPhase =

734

735

736 if (InBootstrapPhase) {

738 [this](LinkGraph &G) { return bootstrapPipelineStart(G); });

740 return bootstrapPipelineRecordRuntimeFunctions(G);

741 });

742 }

743

744

745

747 if (InitSymbol == MP.DSOHandleSymbol && !InBootstrapPhase) {

748 addDSOHandleSupportPasses(MR, Config);

749

750

751 return;

752 }

753

754

756 [this, &MR](jitlink::LinkGraph &G) -> Error {

757 if (auto Err = preserveInitSections(G, MR))

758 return Err;

760 });

761 }

762

763

764 addEHAndTLVSupportPasses(MR, Config, InBootstrapPhase);

765

766

768 InBootstrapPhase](jitlink::LinkGraph &G) {

769 return registerInitSections(G, JD, InBootstrapPhase);

770 });

771

772

773

774 if (InBootstrapPhase)

776 [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });

777}

778

779void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(

780 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {

781

783 jitlink::LinkGraph &G) -> Error {

784 auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {

785 return Sym->getName() == MP.DSOHandleSymbol;

786 });

787 assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");

788 {

789 std::lock_guardstd::mutex Lock(MP.PlatformMutex);

790 auto HandleAddr = (*I)->getAddress();

791 MP.HandleAddrToJITDylib[HandleAddr] = &JD;

792 MP.JITDylibToHandleAddr[&JD] = HandleAddr;

793

794 G.allocActions().push_back(

796 SPSArgList<SPSString, SPSExecutorAddr>>(

797 MP.RegisterJITDylib.Addr, JD.getName(), HandleAddr)),

799 MP.DeregisterJITDylib.Addr, HandleAddr))});

800 }

802 });

803}

804

805void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(

807 bool IsBootstrapping) {

808

809

810

811

812

813

816 return fixTLVSectionsAndEdges(G, JD);

817 });

818

819

820

821 Config.PostFixupPasses.push_back([this, IsBootstrapping](

822 jitlink::LinkGraph &G) -> Error {

823 ELFPerObjectSectionsToRegister POSR;

824

826 jitlink::SectionRange R(*EHFrameSection);

827 if (.empty())

829 }

830

831

832

833 jitlink::Section *ThreadDataSection =

835

836

838

839

840

841 if (ThreadDataSection)

842 G.mergeSections(*ThreadDataSection, *ThreadBSSSection);

843 else

844 ThreadDataSection = ThreadBSSSection;

845 }

846

847

848

849 if (ThreadDataSection) {

850 jitlink::SectionRange R(*ThreadDataSection);

851 if (.empty())

853 }

854

856 if (auto Err = MP.registerPerObjectSections(G, POSR, IsBootstrapping))

857 return Err;

858 }

859

861 });

862}

863

864Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(

865 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {

866

868

869 jitlink::Symbol *InitSym = nullptr;

870

871 for (auto &InitSection : G.sections()) {

872

874 InitSection.empty())

875 continue;

876

877

878

879 if (!InitSym) {

880 auto &B = **InitSection.blocks().begin();

881 InitSym = &G.addDefinedSymbol(

884 }

885

886

887 for (auto *B : InitSection.blocks()) {

889 continue;

890

891 auto &S = G.addAnonymousSymbol(*B, 0, B->getSize(), false, true);

892 InitSym->getBlock().addEdge(jitlink::Edge::KeepAlive, 0, S, 0);

893 }

894 }

895 }

896

898}

899

900Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(

901 jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {

903 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");

904

906 for (auto &Sec : G.sections())

908 OrderedInitSections.push_back(&Sec);

909

910

911

912

913 llvm::sort(OrderedInitSections, [](const jitlink::Section *LHS,

914 const jitlink::Section *RHS) {

917 StringRef LHSPrioStr(LHS->getName());

918 StringRef RHSPrioStr(RHS->getName());

919 uint64_t LHSPriority;

920 bool LHSHasPriority = LHSPrioStr.consume_front(".init_array.") &&

921 !LHSPrioStr.getAsInteger(10, LHSPriority);

922 uint64_t RHSPriority;

923 bool RHSHasPriority = RHSPrioStr.consume_front(".init_array.") &&

924 !RHSPrioStr.getAsInteger(10, RHSPriority);

925 if (LHSHasPriority)

926 return RHSHasPriority ? LHSPriority < RHSPriority : true;

927 else if (RHSHasPriority)

928 return false;

929

930

931 } else {

932

933 return true;

934 }

935 }

937 });

938

939 for (auto &Sec : OrderedInitSections)

940 ELFNixPlatformSecs.push_back(jitlink::SectionRange(*Sec).getRange());

941

942

944 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";

945 for (auto &Sec : G.sections()) {

946 jitlink::SectionRange R(Sec);

947 dbgs() << " " << Sec.getName() << ": " << R.getRange() << "\n";

948 }

949 });

950

951 ExecutorAddr HeaderAddr;

952 {

953 std::lock_guardstd::mutex Lock(MP.PlatformMutex);

954 auto I = MP.JITDylibToHandleAddr.find(&JD);

955 assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");

956 assert(I->second && "Null header registered for JD");

957 HeaderAddr = I->second;

958 }

959

960 using SPSRegisterInitSectionsArgs =

961 SPSArgList<SPSExecutorAddr, SPSSequence>;

962

964 MP.Bootstrap.load()->addArgumentsToRTFnMap(

965 &MP.RegisterInitSections, &MP.DeregisterInitSections,

966 getArgDataBufferType(HeaderAddr,

967 ELFNixPlatformSecs),

968 getArgDataBufferType(HeaderAddr,

969 ELFNixPlatformSecs));

971 }

972

973 G.allocActions().push_back(

975 MP.RegisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs)),

977 MP.DeregisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs))});

978

980}

981

982Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(

983 jitlink::LinkGraph &G, JITDylib &JD) {

984 auto TLSGetAddrSymbolName = G.intern("__tls_get_addr");

985 auto TLSDescResolveSymbolName = G.intern("__tlsdesc_resolver");

986 for (auto *Sym : G.external_symbols()) {

987 if (Sym->getName() == TLSGetAddrSymbolName) {

988 auto TLSGetAddr =

989 MP.getExecutionSession().intern("___orc_rt_elfnix_tls_get_addr");

990 Sym->setName(std::move(TLSGetAddr));

991 } else if (Sym->getName() == TLSDescResolveSymbolName) {

992 auto TLSGetAddr =

993 MP.getExecutionSession().intern("___orc_rt_elfnix_tlsdesc_resolver");

994 Sym->setName(std::move(TLSGetAddr));

995 }

996 }

997

998 auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");

999

1000 if (TLSInfoEntrySection) {

1001 std::optional<uint64_t> Key;

1002 {

1003 std::lock_guardstd::mutex Lock(MP.PlatformMutex);

1004 auto I = MP.JITDylibToPThreadKey.find(&JD);

1005 if (I != MP.JITDylibToPThreadKey.end())

1006 Key = I->second;

1007 }

1008 if ( Key ) {

1009 if (auto KeyOrErr = MP.createPThreadKey())

1010 Key = *KeyOrErr;

1011 else

1012 return KeyOrErr.takeError();

1013 }

1014

1015 uint64_t PlatformKeyBits =

1017

1018 for (auto *B : TLSInfoEntrySection->blocks()) {

1019

1020

1021 assert(B->getSize() == (G.getPointerSize() * 2) &&

1022 "TLS descriptor must be 2 words length");

1023 auto TLSInfoEntryContent = B->getMutableContent(G);

1024 memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());

1025 }

1026 }

1027

1029}

1030

1031}

1032}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

#define LLVM_UNLIKELY(EXPR)

static RegisterPass< DebugifyModulePass > DM("debugify", "Attach debug info to everything")

static StringRef getName(Value *V)

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

iterator find(const_arg_type_t< KeyT > Val)

std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)

size_type count(const_arg_type_t< KeyT > Val) const

Return 1 if the specified key is in the map, 0 otherwise.

void reserve(size_type NumEntries)

Grow the densemap so that it can contain at least NumEntries items before resizing again.

Helper for Errors used as out-parameters.

Lightweight error class with error context and mandatory checking.

static ErrorSuccess success()

Create a success value.

Tagged union holding either a T or a Error.

void push_back(const T &Elt)

pointer data()

Return a pointer to the vector's buffer, even if empty().

StringRef - Represent a constant reference to a string, i.e.

bool starts_with(StringRef Prefix) const

Check if this string starts with the given Prefix.

Manages the enabling and disabling of subtarget specific features.

Triple - Helper class for working with autoconf configuration names.

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

void addEdge(Edge::Kind K, Edge::OffsetT Offset, Symbol &Target, Edge::AddendT Addend)

Add an edge to this block.

const std::string & getName() const

Get the name for this JITLinkDylib.

Block & getBlock()

Return the Block for this Symbol (Symbol must be defined).

Mediates between ELFNix initialization and ExecutionSession state.

ObjectLinkingLayer & getObjectLinkingLayer() const

static Expected< SymbolAliasMap > standardPlatformAliases(ExecutionSession &ES, JITDylib &PlatformJD)

Returns an AliasMap containing the default aliases for the ELFNixPlatform.

Definition ELFNixPlatform.cpp:318

Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) override

This method will be called under the ExecutionSession lock each time a MaterializationUnit is added t...

Definition ELFNixPlatform.cpp:286

Error notifyRemoving(ResourceTracker &RT) override

This method will be called under the ExecutionSession lock when a ResourceTracker is removed.

Definition ELFNixPlatform.cpp:303

Error setupJITDylib(JITDylib &JD) override

This method will be called outside the session lock each time a JITDylib is created (unless it is cre...

Definition ELFNixPlatform.cpp:266

ExecutionSession & getExecutionSession() const

static ArrayRef< std::pair< const char *, const char * > > standardRuntimeUtilityAliases()

Returns the array of standard runtime utility aliases for ELF.

Definition ELFNixPlatform.cpp:337

static Expected< std::unique_ptr< ELFNixPlatform > > Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, std::unique_ptr< DefinitionGenerator > OrcRuntime, std::optional< SymbolAliasMap > RuntimeAliases=std::nullopt)

Try to create a ELFNixPlatform instance, adding the ORC runtime to the given JITDylib.

Definition ELFNixPlatform.cpp:204

static ArrayRef< std::pair< const char *, const char * > > standardLazyCompilationAliases()

Returns a list of aliases required to enable lazy compilation via the ORC runtime.

Definition ELFNixPlatform.cpp:353

static ArrayRef< std::pair< const char *, const char * > > requiredCXXAliases()

Returns the array of required CXX aliases.

Definition ELFNixPlatform.cpp:328

Error teardownJITDylib(JITDylib &JD) override

This method will be called outside the session lock each time a JITDylib is removed to allow the Plat...

Definition ELFNixPlatform.cpp:274

An ExecutionSession represents a running JIT program.

SymbolStringPtr intern(StringRef SymName)

Add a symbol name to the SymbolStringPool and return a pointer to it.

DenseMap< SymbolStringPtr, JITDispatchHandlerFunction > JITDispatchHandlerAssociationMap

A map associating tag names with asynchronous wrapper function implementations in the JIT.

Represents an address in the executor process.

Represents a JIT'd dynamic library.

Error define(std::unique_ptr< MaterializationUnitType > &&MU, ResourceTrackerSP RT=nullptr)

Define all symbols provided by the materialization unit to be part of this JITDylib.

GeneratorT & addGenerator(std::unique_ptr< GeneratorT > DefGenerator)

Adds a definition generator to this JITDylib and returns a referenece to it.

LinkGraphLinkingLayer & addPlugin(std::shared_ptr< Plugin > P)

Add a plugin.

Tracks responsibility for materialization, and mediates interactions between MaterializationUnits and...

const SymbolStringPtr & getInitializerSymbol() const

Returns the initialization pseudo-symbol, if any.

JITDylib & getTargetJITDylib() const

Returns the target JITDylib that these symbols are being materialized into.

A MaterializationUnit represents a set of symbol definitions that can be materialized as a group,...

virtual StringRef getName() const =0

Return the name of this materialization unit.

const SymbolStringPtr & getInitializerSymbol() const

Returns the initialization symbol for this MaterializationUnit (if any).

An ObjectLayer implementation built on JITLink.

void emit(std::unique_ptr< MaterializationResponsibility > R, std::unique_ptr< MemoryBuffer > O) override

Emit an object file.

static void lookupInitSymbolsAsync(unique_function< void(Error)> OnComplete, ExecutionSession &ES, const DenseMap< JITDylib *, SymbolLookupSet > &InitSyms)

Performs an async lookup for the given symbols in each of the given JITDylibs, calling the given hand...

API to remove / transfer ownership of JIT resources.

JITDylib & getJITDylib() const

Return the JITDylib targeted by this tracker.

static Expected< std::unique_ptr< StaticLibraryDefinitionGenerator > > Load(ObjectLayer &L, const char *FileName, VisitMembersFunction VisitMembers=VisitMembersFunction(), GetObjectFileInterface GetObjFileInterface=GetObjectFileInterface())

Try to create a StaticLibraryDefinitionGenerator from the given path.

Pointer to a pooled string representing a symbol name.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ Pointer64

A plain 64-bit pointer value relocation.

@ Pointer64

A plain 64-bit pointer value relocation.

@ Pointer64

A plain 64-bit pointer value relocation.

LLVM_ABI const char * getGenericEdgeKindName(Edge::Kind K)

Returns the string name of the given generic edge kind, or "unknown" otherwise.

LLVM_ABI StringRef ELFThreadBSSSectionName

JITDylibSearchOrder makeJITDylibSearchOrder(ArrayRef< JITDylib * > JDs, JITDylibLookupFlags Flags=JITDylibLookupFlags::MatchExportedSymbolsOnly)

Convenience function for creating a search order from an ArrayRef of JITDylib*, all with the same fla...

std::vector< std::pair< JITDylib *, JITDylibLookupFlags > > JITDylibSearchOrder

A list of (JITDylib*, JITDylibLookupFlags) pairs to be used as a search order during symbol lookup.

IntrusiveRefCntPtr< JITDylib > JITDylibSP

std::vector< std::pair< ExecutorAddr, ELFNixJITDylibDepInfo > > ELFNixJITDylibDepInfoMap

std::unique_ptr< ReExportsMaterializationUnit > symbolAliases(SymbolAliasMap Aliases)

Create a ReExportsMaterializationUnit with the given aliases.

std::vector< ExecutorAddr > ELFNixJITDylibDepInfo

std::unique_ptr< AbsoluteSymbolsMaterializationUnit > absoluteSymbols(SymbolMap Symbols)

Create an AbsoluteSymbolsMaterializationUnit with the given symbols.

LLVM_ABI StringRef ELFEHFrameSectionName

@ MatchExportedSymbolsOnly

static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases, ArrayRef< std::pair< const char *, const char * > > AL)

std::unordered_map< std::pair< RuntimeFunction *, RuntimeFunction * >, SmallVector< std::pair< shared::WrapperFunctionCall::ArgDataBufferType, shared::WrapperFunctionCall::ArgDataBufferType > >, FunctionPairKeyHash, FunctionPairKeyEqual > DeferredRuntimeFnMap

LLVM_ABI StringRef ELFThreadDataSectionName

LLVM_ABI RegisterDependenciesFunction NoDependenciesToRegister

This can be used as the value for a RegisterDependenciesFunction if there are no dependants to regist...

@ Ready

Emitted to memory, but waiting on transitive dependencies.

DenseMap< SymbolStringPtr, SymbolAliasMapEntry > SymbolAliasMap

A map of Symbols to (Symbol, Flags) pairs.

LLVM_ABI bool isELFInitializerSection(StringRef SecName)

DenseMap< SymbolStringPtr, JITSymbolFlags > SymbolFlagsMap

A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags.

value_type byte_swap(value_type value, endianness endian)

This is an optimization pass for GlobalISel generic memory operations.

LLVM_ABI std::error_code inconvertibleErrorCode()

The value returned by this function can be returned from convertToErrorCode for Error values where no...

auto formatv(bool Validate, const char *Fmt, Ts &&...Vals)

void sort(IteratorTy Start, IteratorTy End)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key

Error make_error(ArgTs &&... Args)

Make a Error instance representing failure using the given error info type.

void cantFail(Error Err, const char *Msg=nullptr)

Report a fatal error if Err is a failure value.

OutputIt move(R &&Range, OutputIt Out)

Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.

LinkGraphPassList PostAllocationPasses

Post-allocation passes.

LinkGraphPassList PostFixupPasses

Post-fixup passes.

LinkGraphPassList PostPrunePasses

Post-prune passes.

LinkGraphPassList PrePrunePasses

Pre-prune passes.

ExecutorAddrRange EHFrameSection

ExecutorAddrRange ThreadDataSection