JITLinkMemoryManager.cpp Source File (original) (raw)

14#define DEBUG_TYPE "jitlink"

16using namespace llvm;

18namespace llvm {

26 for (auto &Sec : G.sections()) {

28 if (Sec.blocks().empty() ||

29 Sec.getMemLifetime() == orc::MemLifetime::NoAlloc)

30 continue;

32 auto &Seg = Segments[{Sec.getMemProt(), Sec.getMemLifetime()}];

33 for (auto *B : Sec.blocks())

35 Seg.ContentBlocks.push_back(B);

36 else

37 Seg.ZeroFillBlocks.push_back(B);

38 }

41 auto CompareBlocks = [](const Block *LHS, const Block *RHS) {

43 if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal())

44 return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal();

45 if (LHS->getAddress() != RHS->getAddress())

46 return LHS->getAddress() < RHS->getAddress();

47 return LHS->getSize() < RHS->getSize();

48 };

50 LLVM_DEBUG(dbgs() << "Generated BasicLayout for " << G.getName() << ":\n");

51 for (auto &KV : Segments) {

52 auto &Seg = KV.second;

54 llvm::sort(Seg.ContentBlocks, CompareBlocks);

55 llvm::sort(Seg.ZeroFillBlocks, CompareBlocks);

57 for (auto *B : Seg.ContentBlocks) {

58 Seg.ContentSize = alignToBlock(Seg.ContentSize, *B);

59 Seg.ContentSize += B->getSize();

60 Seg.Alignment = std::max(Seg.Alignment, Align(B->getAlignment()));

61 }

63 uint64_t SegEndOffset = Seg.ContentSize;

64 for (auto *B : Seg.ZeroFillBlocks) {

66 SegEndOffset += B->getSize();

67 Seg.Alignment = std::max(Seg.Alignment, Align(B->getAlignment()));

68 }

69 Seg.ZeroFillSize = SegEndOffset - Seg.ContentSize;

72 dbgs() << " Seg " << KV.first

73 << ": content-size=" << formatv("{0:x}", Seg.ContentSize)

74 << ", zero-fill-size=" << formatv("{0:x}", Seg.ZeroFillSize)

75 << ", align=" << formatv("{0:x}", Seg.Alignment.value()) << "\n";

76 });

77 }

78}

85 auto &AG = KV.first;

86 auto &Seg = KV.second;

95 else

97 }

99 return SegsSizes;

100}

101

103 for (auto &KV : Segments) {

104 auto &Seg = KV.second;

105

106 assert(!(Seg.ContentBlocks.empty() && Seg.ZeroFillBlocks.empty()) &&

107 "Empty section recorded?");

108

109 for (auto *B : Seg.ContentBlocks) {

110

112 Seg.NextWorkingMemOffset = alignToBlock(Seg.NextWorkingMemOffset, *B);

113

114

115 B->setAddress(Seg.Addr);

116 Seg.Addr += B->getSize();

117

118

119

120 memcpy(Seg.WorkingMem + Seg.NextWorkingMemOffset, B->getContent().data(),

121 B->getSize());

122 B->setMutableContent(

123 {Seg.WorkingMem + Seg.NextWorkingMemOffset, B->getSize()});

124 Seg.NextWorkingMemOffset += B->getSize();

125 }

126

127 for (auto *B : Seg.ZeroFillBlocks) {

128

130

131 B->setAddress(Seg.Addr);

132 Seg.Addr += B->getSize();

133 }

134

135 Seg.ContentBlocks.clear();

136 Seg.ZeroFillBlocks.clear();

137 }

138

140}

141

143 return G.allocActions();

144}

145

147 std::shared_ptrorc::SymbolStringPool SSP,

151

153 "AllocGroup has changed. Section names below must be updated");

155 "__---.standard", "__R--.standard", "__-W-.standard", "__RW-.standard",

156 "__--X.standard", "__R-X.standard", "__-WX.standard", "__RWX.standard",

157 "__---.finalize", "__R--.finalize", "__-W-.finalize", "__RW-.finalize",

158 "__--X.finalize", "__R-X.finalize", "__-WX.finalize", "__RWX.finalize"};

159

160 auto G =

161 std::make_unique("", std::move(SSP), std::move(TT),

164

166 for (auto &KV : Segments) {

167 auto &AG = KV.first;

168 auto &Seg = KV.second;

169

171 "NoAlloc segments are not supported by SimpleSegmentAlloc");

172

173 auto AGSectionName =

174 AGSectionNames[static_cast<unsigned>(AG.getMemProt()) |

175 static_cast<bool>(AG.getMemLifetime()) << 3];

176

177 auto &Sec = G->createSection(AGSectionName, AG.getMemProt());

178 Sec.setMemLifetime(AG.getMemLifetime());

179

180 if (Seg.ContentSize != 0) {

181 NextAddr =

183 auto &B =

184 G->createMutableContentBlock(Sec, G->allocateBuffer(Seg.ContentSize),

185 NextAddr, Seg.ContentAlign.value(), 0);

186 ContentBlocks[AG] = &B;

187 NextAddr += Seg.ContentSize;

188 }

189 }

190

191

192 auto &GRef = *G;

194 [G = std::move(G), ContentBlocks = std::move(ContentBlocks),

195 OnCreated = std::move(OnCreated)](

197 if (!Alloc)

198 OnCreated(Alloc.takeError());

199 else

201 std::move(ContentBlocks),

202 std::move(*Alloc)));

203 });

204}

205

209 std::promise<MSVCPExpected> AllocP;

210 auto AllocF = AllocP.get_future();

211 Create(MemMgr, std::move(SSP), std::move(TT), JD, std::move(Segments),

213 AllocP.set_value(std::move(Result));

214 });

215 return AllocF.get();

216}

217

222

225 auto I = ContentBlocks.find(AG);

226 if (I != ContentBlocks.end()) {

227 auto &B = *I->second;

228 return {B.getAddress(), B.getAlreadyMutableContent()};

229 }

230 return {};

231}

232

234 std::unique_ptr G,

236 std::unique_ptrJITLinkMemoryManager::InFlightAlloc Alloc)

239

242public:

246 : MemMgr(MemMgr), G(&G), BL(std::move(BL)),

247 StandardSegments(std::move(StandardSegments)),

248 FinalizationSegments(std::move(FinalizationSegments)) {}

249

251 assert(!G && "InFlight alloc neither abandoned nor finalized");

252 }

253

255

256

257 if (auto Err = applyProtections()) {

258 OnFinalized(std::move(Err));

259 return;

260 }

261

262

263 auto DeallocActions = runFinalizeActions(G->allocActions());

264 if (!DeallocActions) {

265 OnFinalized(DeallocActions.takeError());

266 return;

267 }

268

269

272 return;

273 }

274

275#ifndef NDEBUG

276

277

278

279 G = nullptr;

280#endif

281

282

283 OnFinalized(MemMgr.createFinalizedAlloc(std::move(StandardSegments),

284 std::move(*DeallocActions)));

285 }

286

293

294#ifndef NDEBUG

295

296

297

298 G = nullptr;

299#endif

300

301 OnAbandoned(std::move(Err));

302 }

303

304private:

305 Error applyProtections() {

306 for (auto &KV : BL.segments()) {

307 const auto &AG = KV.first;

308 auto &Seg = KV.second;

309

310 auto Prot = toSysMemoryProtectionFlags(AG.getMemProt());

311

313 alignTo(Seg.ContentSize + Seg.ZeroFillSize, MemMgr.PageSize);

319 }

321 }

322

324 LinkGraph *G;

325 BasicLayout BL;

328};

329

333

336 "Could not create InProcessMemoryManager: Page size " +

337 Twine(*PageSize) + " is not a power of 2",

339

340 return std::make_unique(*PageSize);

341 } else

343}

344

348

349

350

352 if (!SegsSizes) {

353 OnAllocated(SegsSizes.takeError());

354 return;

355 }

356

357

358

359 if (SegsSizes->total() > std::numeric_limits<size_t>::max()) {

361 "Total requested size " + formatv("{0:x}", SegsSizes->total()) +

362 " for graph " + G.getName() + " exceeds address space"));

363 return;

364 }

365

366

367

368

369

370

371

372

373

377 {

381

382 std::error_code EC;

384 ReadWrite, EC);

385

386 if (EC) {

388 return;

389 }

390

391

393

394 StandardSegsMem = {Slab.base(),

395 static_cast<size_t>(SegsSizes->StandardSegs)};

396 FinalizeSegsMem = {(void *)((char *)Slab.base() + SegsSizes->StandardSegs),

397 static_cast<size_t>(SegsSizes->FinalizeSegs)};

398 }

399

402

404 dbgs() << "InProcessMemoryManager allocated:\n";

405 if (SegsSizes->StandardSegs)

406 dbgs() << formatv(" [ {0:x16} -- {1:x16} ]", NextStandardSegAddr,

407 NextStandardSegAddr + StandardSegsMem.allocatedSize())

408 << " to stardard segs\n";

409 else

410 dbgs() << " no standard segs\n";

411 if (SegsSizes->FinalizeSegs)

412 dbgs() << formatv(" [ {0:x16} -- {1:x16} ]", NextFinalizeSegAddr,

413 NextFinalizeSegAddr + FinalizeSegsMem.allocatedSize())

414 << " to finalize segs\n";

415 else

416 dbgs() << " no finalize segs\n";

417 });

418

419

420 for (auto &KV : BL.segments()) {

421 auto &AG = KV.first;

422 auto &Seg = KV.second;

423

425 ? NextStandardSegAddr

426 : NextFinalizeSegAddr;

427

428 Seg.WorkingMem = SegAddr.toPtr<char *>();

429 Seg.Addr = SegAddr;

430

431 SegAddr += alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize);

432 }

433

434 if (auto Err = BL.apply()) {

435 OnAllocated(std::move(Err));

436 return;

437 }

438

439 OnAllocated(std::make_unique(*this, G, std::move(BL),

440 std::move(StandardSegsMem),

441 std::move(FinalizeSegsMem)));

442}

443

446 std::vectorsys::MemoryBlock StandardSegmentsList;

447 std::vector<std::vectororc::shared::WrapperFunctionCall> DeallocActionsList;

448

449 {

450 std::lock_guardstd::mutex Lock(FinalizedAllocsMutex);

451 for (auto &Alloc : Allocs) {

452 auto *FA = Alloc.release().toPtr<FinalizedAllocInfo *>();

453 StandardSegmentsList.push_back(std::move(FA->StandardSegments));

454 DeallocActionsList.push_back(std::move(FA->DeallocActions));

455 FA->~FinalizedAllocInfo();

456 FinalizedAllocInfos.Deallocate(FA);

457 }

458 }

459

461

462 while (!DeallocActionsList.empty()) {

463 auto &DeallocActions = DeallocActionsList.back();

464 auto &StandardSegments = StandardSegmentsList.back();

465

466

467 while (!DeallocActions.empty()) {

468 if (auto Err = DeallocActions.back().runWithSPSRetErrorMerged())

469 DeallocErr = joinErrors(std::move(DeallocErr), std::move(Err));

470 DeallocActions.pop_back();

471 }

472

473

476

477 DeallocActionsList.pop_back();

478 StandardSegmentsList.pop_back();

479 }

480

481 OnDeallocated(std::move(DeallocErr));

482}

483

485InProcessMemoryManager::createFinalizedAlloc(

487 std::vectororc::shared::WrapperFunctionCall DeallocActions) {

488 std::lock_guardstd::mutex Lock(FinalizedAllocsMutex);

489 auto *FA = FinalizedAllocInfos.Allocate();

490 new (FA) FinalizedAllocInfo(

491 {std::move(StandardSegments), std::move(DeallocActions)});

493}

494

495}

496}

for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

#define LLVM_LIKELY(EXPR)

static cl::opt< int > PageSize("imp-null-check-page-size", cl::desc("The page size of the target in bytes"), cl::init(4096), cl::Hidden)

Provides a library for accessing information about this process and other processes on the operating ...

Lightweight error class with error context and mandatory checking.

static ErrorSuccess success()

Create a success value.

Tagged union holding either a T or a Error.

Error takeError()

Take ownership of the stored error.

StringRef - Represent a constant reference to a string, i.e.

Manages the enabling and disabling of subtarget specific features.

Triple - Helper class for working with autoconf configuration names.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

BasicLayout simplifies the implementation of JITLinkMemoryManagers.

LLVM_ABI orc::shared::AllocActions & graphAllocActions()

Returns a reference to the AllocActions in the graph.

Definition JITLinkMemoryManager.cpp:142

LLVM_ABI Error apply()

Apply the layout to the graph.

Definition JITLinkMemoryManager.cpp:102

LLVM_ABI Expected< ContiguousPageBasedLayoutSizes > getContiguousPageBasedLayoutSizes(uint64_t PageSize)

Returns the total number of required to allocate all segments (with each segment padded out to page s...

Definition JITLinkMemoryManager.cpp:81

iterator_range< SegmentMap::iterator > segments()

Returns an iterator over the segments of the layout.

LLVM_ABI BasicLayout(LinkGraph &G)

Definition JITLinkMemoryManager.cpp:24

An Addressable with content and edges.

~IPInFlightAlloc() override

Definition JITLinkMemoryManager.cpp:250

void abandon(OnAbandonedFunction OnAbandoned) override

Called prior to finalization if the allocation should be abandoned.

Definition JITLinkMemoryManager.cpp:287

IPInFlightAlloc(InProcessMemoryManager &MemMgr, LinkGraph &G, BasicLayout BL, sys::MemoryBlock StandardSegments, sys::MemoryBlock FinalizationSegments)

Definition JITLinkMemoryManager.cpp:243

void finalize(OnFinalizedFunction OnFinalized) override

Called to transfer working memory to the target and apply finalization.

Definition JITLinkMemoryManager.cpp:254

static Expected< std::unique_ptr< InProcessMemoryManager > > Create()

Attempts to auto-detect the host page size.

Definition JITLinkMemoryManager.cpp:331

void deallocate(std::vector< FinalizedAlloc > Alloc, OnDeallocatedFunction OnDeallocated) override

Deallocate a list of allocation objects.

Definition JITLinkMemoryManager.cpp:444

void allocate(const JITLinkDylib *JD, LinkGraph &G, OnAllocatedFunction OnAllocated) override

Start the allocation process.

Definition JITLinkMemoryManager.cpp:345

InProcessMemoryManager(uint64_t PageSize)

Create an instance using the given page size.

Represents a finalized allocation.

Represents an allocation which has not been finalized yet.

unique_function< void(Error)> OnAbandonedFunction

unique_function< void(Expected< FinalizedAlloc >)> OnFinalizedFunction

Manages allocations of JIT memory.

virtual void allocate(const JITLinkDylib *JD, LinkGraph &G, OnAllocatedFunction OnAllocated)=0

Start the allocation process.

unique_function< void(AllocResult)> OnAllocatedFunction

Called when allocation has been completed.

unique_function< void(Error)> OnDeallocatedFunction

Called when deallocation has completed.

virtual ~JITLinkMemoryManager()

Expected< std::unique_ptr< InFlightAlloc > > AllocResult

Typedef for the argument to be passed to OnAllocatedFunction.

A utility class for making simple allocations using JITLinkMemoryManager.

unique_function< void(Expected< SimpleSegmentAlloc >)> OnCreatedFunction

LLVM_ABI ~SimpleSegmentAlloc()

orc::AllocGroupSmallMap< Segment > SegmentMap

LLVM_ABI SimpleSegmentAlloc(SimpleSegmentAlloc &&)

LLVM_ABI SegmentInfo getSegInfo(orc::AllocGroup AG)

Returns the SegmentInfo for the given group.

Definition JITLinkMemoryManager.cpp:224

LLVM_ABI SimpleSegmentAlloc & operator=(SimpleSegmentAlloc &&)

static LLVM_ABI void Create(JITLinkMemoryManager &MemMgr, std::shared_ptr< orc::SymbolStringPool > SSP, Triple TT, const JITLinkDylib *JD, SegmentMap Segments, OnCreatedFunction OnCreated)

Definition JITLinkMemoryManager.cpp:146

A specialized small-map for AllocGroups.

A pair of memory protections and allocation policies.

static constexpr unsigned NumGroups

Represents an address in the executor process.

uint64_t getValue() const

static ExecutorAddr fromPtr(T *Ptr, UnwrapFn &&Unwrap=UnwrapFn())

Create an ExecutorAddr from the given pointer.

This class encapsulates the notion of a memory block which has an address and a size.

size_t allocatedSize() const

The size as it was allocated.

static LLVM_ABI std::error_code protectMappedMemory(const MemoryBlock &Block, unsigned Flags)

This method sets the protection flags for a block of memory to the state specified by /p Flags.

static LLVM_ABI std::error_code releaseMappedMemory(MemoryBlock &Block)

This method releases a block of memory that was allocated with the allocateMappedMemory method.

static LLVM_ABI void InvalidateInstructionCache(const void *Addr, size_t Len)

InvalidateInstructionCache - Before the JIT can run a block of code that has been emitted it must inv...

static LLVM_ABI MemoryBlock allocateMappedMemory(size_t NumBytes, const MemoryBlock *const NearBlock, unsigned Flags, std::error_code &EC)

This method allocates a block of memory that is suitable for loading dynamically generated code (e....

static LLVM_ABI Expected< unsigned > getPageSize()

Get the process's page size.

LLVM_ABI const char * getGenericEdgeKindName(Edge::Kind K)

Returns the string name of the given generic edge kind, or "unknown" otherwise.

uint64_t alignToBlock(uint64_t Addr, const Block &B)

@ NoAlloc

NoAlloc memory should not be allocated by the JITLinkMemoryManager at all.

@ Standard

Standard memory should be allocated by the allocator and then deallocated when the deallocate method ...

This is an optimization pass for GlobalISel generic memory operations.

LLVM_ABI std::error_code inconvertibleErrorCode()

The value returned by this function can be returned from convertToErrorCode for Error values where no...

constexpr bool isPowerOf2_64(uint64_t Value)

Return true if the argument is a power of two > 0 (64 bit edition.)

auto formatv(bool Validate, const char *Fmt, Ts &&...Vals)

Error joinErrors(Error E1, Error E2)

Concatenate errors.

void sort(IteratorTy Start, IteratorTy End)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

Error make_error(ArgTs &&... Args)

Make a Error instance representing failure using the given error info type.

uint64_t alignTo(uint64_t Size, Align A)

Returns a multiple of A needed to store Size bytes.

OutputIt move(R &&Range, OutputIt Out)

Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI Error errorCodeToError(std::error_code EC)

Helper for converting an std::error_code to a Error.

Implement std::hash so that hash_code can be used in STL containers.

This struct is a compact representation of a valid (non-zero power of two) alignment.

A convenience class that further groups segments based on memory deallocation policy.

Describes the segment working memory and executor address.