ReOptimizeLayer.cpp Source File (original) (raw)

4using namespace llvm;

5using namespace orc;

7bool ReOptimizeLayer::ReOptMaterializationUnitState::tryStartReoptimize() {

8 std::unique_lockstd::mutex Lock(Mutex);

9 if (Reoptimizing)

10 return false;

12 Reoptimizing = true;

13 return true;

14}

16void ReOptimizeLayer::ReOptMaterializationUnitState::reoptimizeSucceeded() {

17 std::unique_lockstd::mutex Lock(Mutex);

18 assert(Reoptimizing && "Tried to mark unstarted reoptimization as done");

19 Reoptimizing = false;

20 CurVersion++;

21}

23void ReOptimizeLayer::ReOptMaterializationUnitState::reoptimizeFailed() {

24 std::unique_lockstd::mutex Lock(Mutex);

25 assert(Reoptimizing && "Tried to mark unstarted reoptimization as done");

26 Reoptimizing = false;

27}

31 using ReoptimizeSPSSig = shared::SPSError(uint64_t, uint32_t);

32 WFs[Mangle("__orc_rt_reoptimize_tag")] =

33 ES.wrapAsyncWithSPS(this,

34 &ReOptimizeLayer::rt_reoptimize);

35 return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));

36}

40 auto &JD = R->getTargetJITDylib();

42 bool HasNonCallable = false;

43 for (auto &KV : R->getSymbols()) {

44 auto &Flags = KV.second;

45 if (!Flags.isCallable())

46 HasNonCallable = true;

47 }

49 if (HasNonCallable) {

50 BaseLayer.emit(std::move(R), std::move(TSM));

51 return;

52 }

54 auto &MUState = createMaterializationUnitState(TSM);

56 if (auto Err = R->withResourceKeyDo([&](ResourceKey Key) {

57 registerMaterializationUnitResource(Key, MUState);

58 })) {

59 ES.reportError(std::move(Err));

60 R->failMaterialization();

61 return;

62 }

64 if (auto Err =

65 ProfilerFunc(*this, MUState.getID(), MUState.getCurVersion(), TSM)) {

66 ES.reportError(std::move(Err));

67 R->failMaterialization();

68 return;

69 }

71 auto InitialDests =

72 emitMUImplSymbols(MUState, MUState.getCurVersion(), JD, std::move(TSM));

73 if (!InitialDests) {

74 ES.reportError(InitialDests.takeError());

75 R->failMaterialization();

76 return;

77 }

79 RSManager.emitRedirectableSymbols(std::move(R), std::move(*InitialDests));

80}

84 unsigned CurVersion,

91 auto ArgBufferConst = createReoptimizeArgBuffer(M, MUID, CurVersion);

92 if (auto Err = ArgBufferConst.takeError())

93 return Err;

95 new GlobalVariable(M, (*ArgBufferConst)->getType(), true,

97 for (auto &F : M) {

98 if (F.isDeclaration())

99 continue;

100 auto &BB = F.getEntryBlock();

101 auto *IP = &*BB.getFirstInsertionPt();

105

107 Value *Added = IRB.CreateAdd(Cnt, ConstantInt::get(I64Ty, 1));

111 }

113 });

114}

115

117ReOptimizeLayer::emitMUImplSymbols(ReOptMaterializationUnitState &MUState,

122 MangleAndInterner Mangle(ES, M.getDataLayout());

123 for (auto &F : M)

124 if (!F.isDeclaration()) {

125 std::string NewName =

126 (F.getName() + ".__def__." + Twine(Version)).str();

127 RenamedMap[Mangle(F.getName())] = Mangle(NewName);

128 F.setName(NewName);

129 }

131 }));

132

133 auto RT = JD.createResourceTracker();

134 if (auto Err =

135 JD.define(std::make_unique(

137 RT))

138 return Err;

139 MUState.setResourceTracker(RT);

140

142 for (auto [K, V] : RenamedMap)

143 LookupSymbols.add(V);

144

145 auto ImplSymbols =

148 if (auto Err = ImplSymbols.takeError())

149 return Err;

150

152 for (auto [K, V] : RenamedMap)

153 Result[K] = (*ImplSymbols)[V];

154

156}

157

158void ReOptimizeLayer::rt_reoptimize(SendErrorFn SendResult,

160 uint32_t CurVersion) {

161 auto &MUState = getMaterializationUnitState(MUID);

162 if (CurVersion < MUState.getCurVersion() || !MUState.tryStartReoptimize()) {

164 return;

165 }

166

167 ThreadSafeModule TSM = cloneToNewContext(MUState.getThreadSafeModule());

168 auto OldRT = MUState.getResourceTracker();

169 auto &JD = OldRT->getJITDylib();

170

171 if (auto Err = ReOptFunc(*this, MUID, CurVersion + 1, OldRT, TSM)) {

172 ES.reportError(std::move(Err));

173 MUState.reoptimizeFailed();

175 return;

176 }

177

178 auto SymbolDests =

179 emitMUImplSymbols(MUState, CurVersion + 1, JD, std::move(TSM));

180 if (!SymbolDests) {

181 ES.reportError(SymbolDests.takeError());

182 MUState.reoptimizeFailed();

184 return;

185 }

186

187 if (auto Err = RSManager.redirect(JD, std::move(*SymbolDests))) {

188 ES.reportError(std::move(Err));

189 MUState.reoptimizeFailed();

191 return;

192 }

193

194 MUState.reoptimizeSucceeded();

196}

197

198Expected<Constant *> ReOptimizeLayer::createReoptimizeArgBuffer(

200 size_t ArgBufferSize = SPSReoptimizeArgList::size(MUID, CurVersion);

201 std::vector ArgBuffer(ArgBufferSize);

202 shared::SPSOutputBuffer OB(ArgBuffer.data(), ArgBuffer.size());

203 if (!SPSReoptimizeArgList::serialize(OB, MUID, CurVersion))

207}

208

212 M.getGlobalVariable("__orc_rt_jit_dispatch_ctx");

213 if (!DispatchCtx)

216 nullptr, "__orc_rt_jit_dispatch_ctx");

218 M.getGlobalVariable("__orc_rt_reoptimize_tag");

219 if (!ReoptimizeTag)

222 nullptr, "__orc_rt_reoptimize_tag");

224 if (!DispatchFunc) {

225 std::vector<Type *> Args = {PointerType::get(M.getContext(), 0),

232 "__orc_rt_jit_dispatch", &M);

233 }

234 size_t ArgBufferSizeConst =

236 Constant *ArgBufferSize = ConstantInt::get(

237 IntegerType::get(M.getContext(), 64), ArgBufferSizeConst, false);

240 {DispatchCtx, ReoptimizeTag, ArgBuffer, ArgBufferSize});

241}

242

243ReOptimizeLayer::ReOptMaterializationUnitState &

244ReOptimizeLayer::createMaterializationUnitState(const ThreadSafeModule &TSM) {

245 std::unique_lockstd::mutex Lock(Mutex);

247 MUStates.emplace(MUID,

249 ++NextID;

250 return MUStates.at(MUID);

251}

252

253ReOptimizeLayer::ReOptMaterializationUnitState &

255 std::unique_lockstd::mutex Lock(Mutex);

256 return MUStates.at(MUID);

257}

258

259void ReOptimizeLayer::registerMaterializationUnitResource(

260 ResourceKey Key, ReOptMaterializationUnitState &State) {

261 std::unique_lockstd::mutex Lock(Mutex);

262 MUResources[Key].insert(State.getID());

263}

264

266 std::unique_lockstd::mutex Lock(Mutex);

267 for (auto MUID : MUResources[K])

268 MUStates.erase(MUID);

269

270 MUResources.erase(K);

272}

273

276 std::unique_lockstd::mutex Lock(Mutex);

277 MUResources[DstK].insert_range(MUResources[SrcK]);

278 MUResources.erase(SrcK);

279}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

Machine Check Debug Module

static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)

get() constructor - Return a constant with array type with an element count and element type matching...

This is an important base class in LLVM.

static LLVM_ABI Constant * getNullValue(Type *Ty)

Constructor to create a '0' constant of arbitrary type.

Lightweight error class with error context and mandatory checking.

static ErrorSuccess success()

Create a success value.

Tagged union holding either a T or a Error.

Class to represent function types.

static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)

const Function & getFunction() const

@ InternalLinkage

Rename collisions when linking (static functions).

@ ExternalLinkage

Externally visible function.

Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")

LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)

Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...

StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)

Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)

This static method is the primary way of constructing an IntegerType.

A Module instance is used to store all the information related to an LLVM module.

static LLVM_ABI PointerType * get(Type *ElementType, unsigned AddressSpace)

This constructs a pointer to an object of the specified type in a numbered address space.

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)

static LLVM_ABI Type * getVoidTy(LLVMContext &C)

LLVM Value Representation.

LLVM_ABI void lookup(LookupKind K, const JITDylibSearchOrder &SearchOrder, SymbolLookupSet Symbols, SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete, RegisterDependenciesFunction RegisterDependencies)

Search the given JITDylibs for the given symbols.

DenseMap< SymbolStringPtr, JITDispatchHandlerFunction > JITDispatchHandlerAssociationMap

A map associating tag names with asynchronous wrapper function implementations in the JIT.

const IRSymbolMapper::ManglingOptions *& getManglingOptions() const

Get the mangling options for this layer.

Represents a JIT'd dynamic library.

ReOptimizeLayer(ExecutionSession &ES, DataLayout &DL, IRLayer &BaseLayer, RedirectableSymbolManager &RM)

void emit(std::unique_ptr< MaterializationResponsibility > R, ThreadSafeModule TSM) override

Emits the given module.

Definition ReOptimizeLayer.cpp:38

Error reigsterRuntimeFunctions(JITDylib &PlatformJD)

Registers reoptimize runtime dispatch handlers to given PlatformJD.

Definition ReOptimizeLayer.cpp:29

static void createReoptimizeCall(Module &M, Instruction &IP, GlobalVariable *ArgBuffer)

Definition ReOptimizeLayer.cpp:209

uint64_t ReOptMaterializationUnitID

void handleTransferResources(JITDylib &JD, ResourceKey DstK, ResourceKey SrcK) override

This function will be called inside the session lock.

Definition ReOptimizeLayer.cpp:274

static Error reoptimizeIfCallFrequent(ReOptimizeLayer &Parent, ReOptMaterializationUnitID MUID, unsigned CurVersion, ThreadSafeModule &TSM)

Basic AddProfilerFunc that reoptimizes the function when the call count exceeds CallCountThreshold.

Definition ReOptimizeLayer.cpp:82

Error handleRemoveResources(JITDylib &JD, ResourceKey K) override

This function will be called outside the session lock.

Definition ReOptimizeLayer.cpp:265

static const uint64_t CallCountThreshold

A set of symbols to look up, each associated with a SymbolLookupFlags value.

SymbolLookupSet & add(SymbolStringPtr Name, SymbolLookupFlags Flags=SymbolLookupFlags::RequiredSymbol)

Add an element to the set.

An LLVM Module together with a shared ThreadSafeContext.

decltype(auto) withModuleDo(Func &&F)

Locks the associated ThreadSafeContext and calls the given function on the contained Module.

@ OB

OB - OneByte - Set if this instruction has a one byte opcode.

DenseMap< SymbolStringPtr, ExecutorSymbolDef > SymbolMap

A map from symbol names (as SymbolStringPtrs) to JITSymbols (address/flags pairs).

LLVM_ABI ThreadSafeModule cloneToNewContext(const ThreadSafeModule &TSMW, GVPredicate ShouldCloneDef=GVPredicate(), GVModifier UpdateClonedDefSource=GVModifier())

Clones the given module on to a new context.

@ Resolved

Queried, materialization begun.

SmartMutex< false > Mutex

Mutex - A standard, always enforced mutex.

This is an optimization pass for GlobalISel generic memory operations.

LLVM_ABI std::error_code inconvertibleErrorCode()

The value returned by this function can be returned from convertToErrorCode for Error values where no...

FunctionAddr VTableAddr uintptr_t uintptr_t Version

LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key

Error make_error(ArgTs &&... Args)

Make a Error instance representing failure using the given error info type.

void cantFail(Error Err, const char *Msg=nullptr)

Report a fatal error if Err is a failure value.

ArrayRef(const T &OneElt) -> ArrayRef< T >

LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)

Split the containing block at the specified instruction - everything before SplitBefore stays in the ...