565 lines
20 KiB
C
565 lines
20 KiB
C
|
//===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===//
|
||
|
|
//
|
||
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
||
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||
|
|
//
|
||
|
|
//===----------------------------------------------------------------------===//
|
||
|
|
//
|
||
|
|
// Contains utilities for adding indirections and breaking up modules.
|
||
|
|
//
|
||
|
|
//===----------------------------------------------------------------------===//
|
||
|
|
|
||
|
|
#ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
|
||
|
|
#define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
|
||
|
|
|
||
|
|
#include "llvm/ADT/StringMap.h"
|
||
|
|
#include "llvm/ADT/StringRef.h"
|
||
|
|
#include "llvm/ExecutionEngine/JITSymbol.h"
|
||
|
|
#include "llvm/ExecutionEngine/Orc/Core.h"
|
||
|
|
#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
|
||
|
|
#include "llvm/Support/Error.h"
|
||
|
|
#include "llvm/Support/Memory.h"
|
||
|
|
#include "llvm/Support/Process.h"
|
||
|
|
#include "llvm/Transforms/Utils/ValueMapper.h"
|
||
|
|
#include <algorithm>
|
||
|
|
#include <cassert>
|
||
|
|
#include <cstdint>
|
||
|
|
#include <functional>
|
||
|
|
#include <future>
|
||
|
|
#include <map>
|
||
|
|
#include <memory>
|
||
|
|
#include <system_error>
|
||
|
|
#include <utility>
|
||
|
|
#include <vector>
|
||
|
|
|
||
|
|
namespace llvm {
|
||
|
|
|
||
|
|
class Constant;
|
||
|
|
class Function;
|
||
|
|
class FunctionType;
|
||
|
|
class GlobalAlias;
|
||
|
|
class GlobalVariable;
|
||
|
|
class Module;
|
||
|
|
class PointerType;
|
||
|
|
class Triple;
|
||
|
|
class Twine;
|
||
|
|
class Value;
|
||
|
|
|
||
|
|
namespace orc {
|
||
|
|
|
||
|
|
/// Base class for pools of compiler re-entry trampolines.
|
||
|
|
/// These trampolines are callable addresses that save all register state
|
||
|
|
/// before calling a supplied function to return the trampoline landing
|
||
|
|
/// address, then restore all state before jumping to that address. They
|
||
|
|
/// are used by various ORC APIs to support lazy compilation
|
||
|
|
class TrampolinePool {
|
||
|
|
public:
|
||
|
|
using NotifyLandingResolvedFunction =
|
||
|
|
unique_function<void(JITTargetAddress) const>;
|
||
|
|
|
||
|
|
using ResolveLandingFunction = unique_function<void(
|
||
|
|
JITTargetAddress TrampolineAddr,
|
||
|
|
NotifyLandingResolvedFunction OnLandingResolved) const>;
|
||
|
|
|
||
|
|
virtual ~TrampolinePool();
|
||
|
|
|
||
|
|
/// Get an available trampoline address.
|
||
|
|
/// Returns an error if no trampoline can be created.
|
||
|
|
Expected<JITTargetAddress> getTrampoline() {
|
||
|
|
std::lock_guard<std::mutex> Lock(TPMutex);
|
||
|
|
if (AvailableTrampolines.empty()) {
|
||
|
|
if (auto Err = grow())
|
||
|
|
return std::move(Err);
|
||
|
|
}
|
||
|
|
assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");
|
||
|
|
auto TrampolineAddr = AvailableTrampolines.back();
|
||
|
|
AvailableTrampolines.pop_back();
|
||
|
|
return TrampolineAddr;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Returns the given trampoline to the pool for re-use.
|
||
|
|
void releaseTrampoline(JITTargetAddress TrampolineAddr) {
|
||
|
|
std::lock_guard<std::mutex> Lock(TPMutex);
|
||
|
|
AvailableTrampolines.push_back(TrampolineAddr);
|
||
|
|
}
|
||
|
|
|
||
|
|
protected:
|
||
|
|
virtual Error grow() = 0;
|
||
|
|
|
||
|
|
std::mutex TPMutex;
|
||
|
|
std::vector<JITTargetAddress> AvailableTrampolines;
|
||
|
|
};
|
||
|
|
|
||
|
|
/// A trampoline pool for trampolines within the current process.
|
||
|
|
template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
|
||
|
|
public:
|
||
|
|
/// Creates a LocalTrampolinePool with the given RunCallback function.
|
||
|
|
/// Returns an error if this function is unable to correctly allocate, write
|
||
|
|
/// and protect the resolver code block.
|
||
|
|
static Expected<std::unique_ptr<LocalTrampolinePool>>
|
||
|
|
Create(ResolveLandingFunction ResolveLanding) {
|
||
|
|
Error Err = Error::success();
|
||
|
|
|
||
|
|
auto LTP = std::unique_ptr<LocalTrampolinePool>(
|
||
|
|
new LocalTrampolinePool(std::move(ResolveLanding), Err));
|
||
|
|
|
||
|
|
if (Err)
|
||
|
|
return std::move(Err);
|
||
|
|
return std::move(LTP);
|
||
|
|
}
|
||
|
|
|
||
|
|
private:
|
||
|
|
static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) {
|
||
|
|
LocalTrampolinePool<ORCABI> *TrampolinePool =
|
||
|
|
static_cast<LocalTrampolinePool *>(TrampolinePoolPtr);
|
||
|
|
|
||
|
|
std::promise<JITTargetAddress> LandingAddressP;
|
||
|
|
auto LandingAddressF = LandingAddressP.get_future();
|
||
|
|
|
||
|
|
TrampolinePool->ResolveLanding(pointerToJITTargetAddress(TrampolineId),
|
||
|
|
[&](JITTargetAddress LandingAddress) {
|
||
|
|
LandingAddressP.set_value(LandingAddress);
|
||
|
|
});
|
||
|
|
return LandingAddressF.get();
|
||
|
|
}
|
||
|
|
|
||
|
|
LocalTrampolinePool(ResolveLandingFunction ResolveLanding, Error &Err)
|
||
|
|
: ResolveLanding(std::move(ResolveLanding)) {
|
||
|
|
|
||
|
|
ErrorAsOutParameter _(&Err);
|
||
|
|
|
||
|
|
/// Try to set up the resolver block.
|
||
|
|
std::error_code EC;
|
||
|
|
ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
|
||
|
|
ORCABI::ResolverCodeSize, nullptr,
|
||
|
|
sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
|
||
|
|
if (EC) {
|
||
|
|
Err = errorCodeToError(EC);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
ORCABI::writeResolverCode(static_cast<char *>(ResolverBlock.base()),
|
||
|
|
pointerToJITTargetAddress(ResolverBlock.base()),
|
||
|
|
pointerToJITTargetAddress(&reenter),
|
||
|
|
pointerToJITTargetAddress(this));
|
||
|
|
|
||
|
|
EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(),
|
||
|
|
sys::Memory::MF_READ |
|
||
|
|
sys::Memory::MF_EXEC);
|
||
|
|
if (EC) {
|
||
|
|
Err = errorCodeToError(EC);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
Error grow() override {
|
||
|
|
assert(AvailableTrampolines.empty() && "Growing prematurely?");
|
||
|
|
|
||
|
|
std::error_code EC;
|
||
|
|
auto TrampolineBlock =
|
||
|
|
sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
|
||
|
|
sys::Process::getPageSizeEstimate(), nullptr,
|
||
|
|
sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
|
||
|
|
if (EC)
|
||
|
|
return errorCodeToError(EC);
|
||
|
|
|
||
|
|
unsigned NumTrampolines =
|
||
|
|
(sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) /
|
||
|
|
ORCABI::TrampolineSize;
|
||
|
|
|
||
|
|
char *TrampolineMem = static_cast<char *>(TrampolineBlock.base());
|
||
|
|
ORCABI::writeTrampolines(
|
||
|
|
TrampolineMem, pointerToJITTargetAddress(TrampolineMem),
|
||
|
|
pointerToJITTargetAddress(ResolverBlock.base()), NumTrampolines);
|
||
|
|
|
||
|
|
for (unsigned I = 0; I < NumTrampolines; ++I)
|
||
|
|
AvailableTrampolines.push_back(pointerToJITTargetAddress(
|
||
|
|
TrampolineMem + (I * ORCABI::TrampolineSize)));
|
||
|
|
|
||
|
|
if (auto EC = sys::Memory::protectMappedMemory(
|
||
|
|
TrampolineBlock.getMemoryBlock(),
|
||
|
|
sys::Memory::MF_READ | sys::Memory::MF_EXEC))
|
||
|
|
return errorCodeToError(EC);
|
||
|
|
|
||
|
|
TrampolineBlocks.push_back(std::move(TrampolineBlock));
|
||
|
|
return Error::success();
|
||
|
|
}
|
||
|
|
|
||
|
|
ResolveLandingFunction ResolveLanding;
|
||
|
|
|
||
|
|
sys::OwningMemoryBlock ResolverBlock;
|
||
|
|
std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
|
||
|
|
};
|
||
|
|
|
||
|
|
/// Target-independent base class for compile callback management.
|
||
|
|
class JITCompileCallbackManager {
|
||
|
|
public:
|
||
|
|
using CompileFunction = std::function<JITTargetAddress()>;
|
||
|
|
|
||
|
|
virtual ~JITCompileCallbackManager() = default;
|
||
|
|
|
||
|
|
/// Reserve a compile callback.
|
||
|
|
Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile);
|
||
|
|
|
||
|
|
/// Execute the callback for the given trampoline id. Called by the JIT
|
||
|
|
/// to compile functions on demand.
|
||
|
|
JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr);
|
||
|
|
|
||
|
|
protected:
|
||
|
|
/// Construct a JITCompileCallbackManager.
|
||
|
|
JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP,
|
||
|
|
ExecutionSession &ES,
|
||
|
|
JITTargetAddress ErrorHandlerAddress)
|
||
|
|
: TP(std::move(TP)), ES(ES),
|
||
|
|
CallbacksJD(ES.createBareJITDylib("<Callbacks>")),
|
||
|
|
ErrorHandlerAddress(ErrorHandlerAddress) {}
|
||
|
|
|
||
|
|
void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) {
|
||
|
|
this->TP = std::move(TP);
|
||
|
|
}
|
||
|
|
|
||
|
|
private:
|
||
|
|
std::mutex CCMgrMutex;
|
||
|
|
std::unique_ptr<TrampolinePool> TP;
|
||
|
|
ExecutionSession &ES;
|
||
|
|
JITDylib &CallbacksJD;
|
||
|
|
JITTargetAddress ErrorHandlerAddress;
|
||
|
|
std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol;
|
||
|
|
size_t NextCallbackId = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
/// Manage compile callbacks for in-process JITs.
|
||
|
|
template <typename ORCABI>
|
||
|
|
class LocalJITCompileCallbackManager : public JITCompileCallbackManager {
|
||
|
|
public:
|
||
|
|
/// Create a new LocalJITCompileCallbackManager.
|
||
|
|
static Expected<std::unique_ptr<LocalJITCompileCallbackManager>>
|
||
|
|
Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) {
|
||
|
|
Error Err = Error::success();
|
||
|
|
auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>(
|
||
|
|
new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err));
|
||
|
|
if (Err)
|
||
|
|
return std::move(Err);
|
||
|
|
return std::move(CCMgr);
|
||
|
|
}
|
||
|
|
|
||
|
|
private:
|
||
|
|
/// Construct a InProcessJITCompileCallbackManager.
|
||
|
|
/// @param ErrorHandlerAddress The address of an error handler in the target
|
||
|
|
/// process to be used if a compile callback fails.
|
||
|
|
LocalJITCompileCallbackManager(ExecutionSession &ES,
|
||
|
|
JITTargetAddress ErrorHandlerAddress,
|
||
|
|
Error &Err)
|
||
|
|
: JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) {
|
||
|
|
using NotifyLandingResolvedFunction =
|
||
|
|
TrampolinePool::NotifyLandingResolvedFunction;
|
||
|
|
|
||
|
|
ErrorAsOutParameter _(&Err);
|
||
|
|
auto TP = LocalTrampolinePool<ORCABI>::Create(
|
||
|
|
[this](JITTargetAddress TrampolineAddr,
|
||
|
|
NotifyLandingResolvedFunction NotifyLandingResolved) {
|
||
|
|
NotifyLandingResolved(executeCompileCallback(TrampolineAddr));
|
||
|
|
});
|
||
|
|
|
||
|
|
if (!TP) {
|
||
|
|
Err = TP.takeError();
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
setTrampolinePool(std::move(*TP));
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
/// Base class for managing collections of named indirect stubs.
|
||
|
|
class IndirectStubsManager {
|
||
|
|
public:
|
||
|
|
/// Map type for initializing the manager. See init.
|
||
|
|
using StubInitsMap = StringMap<std::pair<JITTargetAddress, JITSymbolFlags>>;
|
||
|
|
|
||
|
|
virtual ~IndirectStubsManager() = default;
|
||
|
|
|
||
|
|
/// Create a single stub with the given name, target address and flags.
|
||
|
|
virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr,
|
||
|
|
JITSymbolFlags StubFlags) = 0;
|
||
|
|
|
||
|
|
/// Create StubInits.size() stubs with the given names, target
|
||
|
|
/// addresses, and flags.
|
||
|
|
virtual Error createStubs(const StubInitsMap &StubInits) = 0;
|
||
|
|
|
||
|
|
/// Find the stub with the given name. If ExportedStubsOnly is true,
|
||
|
|
/// this will only return a result if the stub's flags indicate that it
|
||
|
|
/// is exported.
|
||
|
|
virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0;
|
||
|
|
|
||
|
|
/// Find the implementation-pointer for the stub.
|
||
|
|
virtual JITEvaluatedSymbol findPointer(StringRef Name) = 0;
|
||
|
|
|
||
|
|
/// Change the value of the implementation pointer for the stub.
|
||
|
|
virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0;
|
||
|
|
|
||
|
|
private:
|
||
|
|
virtual void anchor();
|
||
|
|
};
|
||
|
|
|
||
|
|
template <typename ORCABI> class LocalIndirectStubsInfo {
|
||
|
|
public:
|
||
|
|
LocalIndirectStubsInfo(unsigned NumStubs, sys::OwningMemoryBlock StubsMem)
|
||
|
|
: NumStubs(NumStubs), StubsMem(std::move(StubsMem)) {}
|
||
|
|
|
||
|
|
static Expected<LocalIndirectStubsInfo> create(unsigned MinStubs,
|
||
|
|
unsigned PageSize) {
|
||
|
|
auto ISAS = getIndirectStubsBlockSizes<ORCABI>(MinStubs, PageSize);
|
||
|
|
|
||
|
|
assert((ISAS.StubBytes % PageSize == 0) &&
|
||
|
|
"StubBytes is not a page size multiple");
|
||
|
|
uint64_t PointerAlloc = alignTo(ISAS.PointerBytes, PageSize);
|
||
|
|
|
||
|
|
// Allocate memory for stubs and pointers in one call.
|
||
|
|
std::error_code EC;
|
||
|
|
auto StubsAndPtrsMem =
|
||
|
|
sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
|
||
|
|
ISAS.StubBytes + PointerAlloc, nullptr,
|
||
|
|
sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
|
||
|
|
if (EC)
|
||
|
|
return errorCodeToError(EC);
|
||
|
|
|
||
|
|
sys::MemoryBlock StubsBlock(StubsAndPtrsMem.base(), ISAS.StubBytes);
|
||
|
|
auto StubsBlockMem = static_cast<char *>(StubsAndPtrsMem.base());
|
||
|
|
auto PtrBlockAddress =
|
||
|
|
pointerToJITTargetAddress(StubsBlockMem) + ISAS.StubBytes;
|
||
|
|
|
||
|
|
ORCABI::writeIndirectStubsBlock(StubsBlockMem,
|
||
|
|
pointerToJITTargetAddress(StubsBlockMem),
|
||
|
|
PtrBlockAddress, ISAS.NumStubs);
|
||
|
|
|
||
|
|
if (auto EC = sys::Memory::protectMappedMemory(
|
||
|
|
StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC))
|
||
|
|
return errorCodeToError(EC);
|
||
|
|
|
||
|
|
return LocalIndirectStubsInfo(ISAS.NumStubs, std::move(StubsAndPtrsMem));
|
||
|
|
}
|
||
|
|
|
||
|
|
unsigned getNumStubs() const { return NumStubs; }
|
||
|
|
|
||
|
|
void *getStub(unsigned Idx) const {
|
||
|
|
return static_cast<char *>(StubsMem.base()) + Idx * ORCABI::StubSize;
|
||
|
|
}
|
||
|
|
|
||
|
|
void **getPtr(unsigned Idx) const {
|
||
|
|
char *PtrsBase =
|
||
|
|
static_cast<char *>(StubsMem.base()) + NumStubs * ORCABI::StubSize;
|
||
|
|
return reinterpret_cast<void **>(PtrsBase) + Idx;
|
||
|
|
}
|
||
|
|
|
||
|
|
private:
|
||
|
|
unsigned NumStubs = 0;
|
||
|
|
sys::OwningMemoryBlock StubsMem;
|
||
|
|
};
|
||
|
|
|
||
|
|
/// IndirectStubsManager implementation for the host architecture, e.g.
|
||
|
|
/// OrcX86_64. (See OrcArchitectureSupport.h).
|
||
|
|
template <typename TargetT>
|
||
|
|
class LocalIndirectStubsManager : public IndirectStubsManager {
|
||
|
|
public:
|
||
|
|
Error createStub(StringRef StubName, JITTargetAddress StubAddr,
|
||
|
|
JITSymbolFlags StubFlags) override {
|
||
|
|
std::lock_guard<std::mutex> Lock(StubsMutex);
|
||
|
|
if (auto Err = reserveStubs(1))
|
||
|
|
return Err;
|
||
|
|
|
||
|
|
createStubInternal(StubName, StubAddr, StubFlags);
|
||
|
|
|
||
|
|
return Error::success();
|
||
|
|
}
|
||
|
|
|
||
|
|
Error createStubs(const StubInitsMap &StubInits) override {
|
||
|
|
std::lock_guard<std::mutex> Lock(StubsMutex);
|
||
|
|
if (auto Err = reserveStubs(StubInits.size()))
|
||
|
|
return Err;
|
||
|
|
|
||
|
|
for (auto &Entry : StubInits)
|
||
|
|
createStubInternal(Entry.first(), Entry.second.first,
|
||
|
|
Entry.second.second);
|
||
|
|
|
||
|
|
return Error::success();
|
||
|
|
}
|
||
|
|
|
||
|
|
JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override {
|
||
|
|
std::lock_guard<std::mutex> Lock(StubsMutex);
|
||
|
|
auto I = StubIndexes.find(Name);
|
||
|
|
if (I == StubIndexes.end())
|
||
|
|
return nullptr;
|
||
|
|
auto Key = I->second.first;
|
||
|
|
void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second);
|
||
|
|
assert(StubAddr && "Missing stub address");
|
||
|
|
auto StubTargetAddr =
|
||
|
|
static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr));
|
||
|
|
auto StubSymbol = JITEvaluatedSymbol(StubTargetAddr, I->second.second);
|
||
|
|
if (ExportedStubsOnly && !StubSymbol.getFlags().isExported())
|
||
|
|
return nullptr;
|
||
|
|
return StubSymbol;
|
||
|
|
}
|
||
|
|
|
||
|
|
JITEvaluatedSymbol findPointer(StringRef Name) override {
|
||
|
|
std::lock_guard<std::mutex> Lock(StubsMutex);
|
||
|
|
auto I = StubIndexes.find(Name);
|
||
|
|
if (I == StubIndexes.end())
|
||
|
|
return nullptr;
|
||
|
|
auto Key = I->second.first;
|
||
|
|
void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second);
|
||
|
|
assert(PtrAddr && "Missing pointer address");
|
||
|
|
auto PtrTargetAddr =
|
||
|
|
static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr));
|
||
|
|
return JITEvaluatedSymbol(PtrTargetAddr, I->second.second);
|
||
|
|
}
|
||
|
|
|
||
|
|
Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override {
|
||
|
|
using AtomicIntPtr = std::atomic<uintptr_t>;
|
||
|
|
|
||
|
|
std::lock_guard<std::mutex> Lock(StubsMutex);
|
||
|
|
auto I = StubIndexes.find(Name);
|
||
|
|
assert(I != StubIndexes.end() && "No stub pointer for symbol");
|
||
|
|
auto Key = I->second.first;
|
||
|
|
AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>(
|
||
|
|
IndirectStubsInfos[Key.first].getPtr(Key.second));
|
||
|
|
*AtomicStubPtr = static_cast<uintptr_t>(NewAddr);
|
||
|
|
return Error::success();
|
||
|
|
}
|
||
|
|
|
||
|
|
private:
|
||
|
|
Error reserveStubs(unsigned NumStubs) {
|
||
|
|
if (NumStubs <= FreeStubs.size())
|
||
|
|
return Error::success();
|
||
|
|
|
||
|
|
unsigned NewStubsRequired = NumStubs - FreeStubs.size();
|
||
|
|
unsigned NewBlockId = IndirectStubsInfos.size();
|
||
|
|
auto ISI =
|
||
|
|
LocalIndirectStubsInfo<TargetT>::create(NewStubsRequired, PageSize);
|
||
|
|
if (!ISI)
|
||
|
|
return ISI.takeError();
|
||
|
|
for (unsigned I = 0; I < ISI->getNumStubs(); ++I)
|
||
|
|
FreeStubs.push_back(std::make_pair(NewBlockId, I));
|
||
|
|
IndirectStubsInfos.push_back(std::move(*ISI));
|
||
|
|
return Error::success();
|
||
|
|
}
|
||
|
|
|
||
|
|
void createStubInternal(StringRef StubName, JITTargetAddress InitAddr,
|
||
|
|
JITSymbolFlags StubFlags) {
|
||
|
|
auto Key = FreeStubs.back();
|
||
|
|
FreeStubs.pop_back();
|
||
|
|
*IndirectStubsInfos[Key.first].getPtr(Key.second) =
|
||
|
|
jitTargetAddressToPointer<void *>(InitAddr);
|
||
|
|
StubIndexes[StubName] = std::make_pair(Key, StubFlags);
|
||
|
|
}
|
||
|
|
|
||
|
|
unsigned PageSize = sys::Process::getPageSizeEstimate();
|
||
|
|
std::mutex StubsMutex;
|
||
|
|
std::vector<LocalIndirectStubsInfo<TargetT>> IndirectStubsInfos;
|
||
|
|
using StubKey = std::pair<uint16_t, uint16_t>;
|
||
|
|
std::vector<StubKey> FreeStubs;
|
||
|
|
StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes;
|
||
|
|
};
|
||
|
|
|
||
|
|
/// Create a local compile callback manager.
|
||
|
|
///
|
||
|
|
/// The given target triple will determine the ABI, and the given
|
||
|
|
/// ErrorHandlerAddress will be used by the resulting compile callback
|
||
|
|
/// manager if a compile callback fails.
|
||
|
|
Expected<std::unique_ptr<JITCompileCallbackManager>>
|
||
|
|
createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
|
||
|
|
JITTargetAddress ErrorHandlerAddress);
|
||
|
|
|
||
|
|
/// Create a local indriect stubs manager builder.
|
||
|
|
///
|
||
|
|
/// The given target triple will determine the ABI.
|
||
|
|
std::function<std::unique_ptr<IndirectStubsManager>()>
|
||
|
|
createLocalIndirectStubsManagerBuilder(const Triple &T);
|
||
|
|
|
||
|
|
/// Build a function pointer of FunctionType with the given constant
|
||
|
|
/// address.
|
||
|
|
///
|
||
|
|
/// Usage example: Turn a trampoline address into a function pointer constant
|
||
|
|
/// for use in a stub.
|
||
|
|
Constant *createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr);
|
||
|
|
|
||
|
|
/// Create a function pointer with the given type, name, and initializer
|
||
|
|
/// in the given Module.
|
||
|
|
GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name,
|
||
|
|
Constant *Initializer);
|
||
|
|
|
||
|
|
/// Turn a function declaration into a stub function that makes an
|
||
|
|
/// indirect call using the given function pointer.
|
||
|
|
void makeStub(Function &F, Value &ImplPointer);
|
||
|
|
|
||
|
|
/// Promotes private symbols to global hidden, and renames to prevent clashes
|
||
|
|
/// with other promoted symbols. The same SymbolPromoter instance should be
|
||
|
|
/// used for all symbols to be added to a single JITDylib.
|
||
|
|
class SymbolLinkagePromoter {
|
||
|
|
public:
|
||
|
|
/// Promote symbols in the given module. Returns the set of global values
|
||
|
|
/// that have been renamed/promoted.
|
||
|
|
std::vector<GlobalValue *> operator()(Module &M);
|
||
|
|
|
||
|
|
private:
|
||
|
|
unsigned NextId = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
/// Clone a function declaration into a new module.
|
||
|
|
///
|
||
|
|
/// This function can be used as the first step towards creating a callback
|
||
|
|
/// stub (see makeStub), or moving a function body (see moveFunctionBody).
|
||
|
|
///
|
||
|
|
/// If the VMap argument is non-null, a mapping will be added between F and
|
||
|
|
/// the new declaration, and between each of F's arguments and the new
|
||
|
|
/// declaration's arguments. This map can then be passed in to moveFunction to
|
||
|
|
/// move the function body if required. Note: When moving functions between
|
||
|
|
/// modules with these utilities, all decls should be cloned (and added to a
|
||
|
|
/// single VMap) before any bodies are moved. This will ensure that references
|
||
|
|
/// between functions all refer to the versions in the new module.
|
||
|
|
Function *cloneFunctionDecl(Module &Dst, const Function &F,
|
||
|
|
ValueToValueMapTy *VMap = nullptr);
|
||
|
|
|
||
|
|
/// Move the body of function 'F' to a cloned function declaration in a
|
||
|
|
/// different module (See related cloneFunctionDecl).
|
||
|
|
///
|
||
|
|
/// If the target function declaration is not supplied via the NewF parameter
|
||
|
|
/// then it will be looked up via the VMap.
|
||
|
|
///
|
||
|
|
/// This will delete the body of function 'F' from its original parent module,
|
||
|
|
/// but leave its declaration.
|
||
|
|
void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
|
||
|
|
ValueMaterializer *Materializer = nullptr,
|
||
|
|
Function *NewF = nullptr);
|
||
|
|
|
||
|
|
/// Clone a global variable declaration into a new module.
|
||
|
|
GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
|
||
|
|
ValueToValueMapTy *VMap = nullptr);
|
||
|
|
|
||
|
|
/// Move global variable GV from its parent module to cloned global
|
||
|
|
/// declaration in a different module.
|
||
|
|
///
|
||
|
|
/// If the target global declaration is not supplied via the NewGV parameter
|
||
|
|
/// then it will be looked up via the VMap.
|
||
|
|
///
|
||
|
|
/// This will delete the initializer of GV from its original parent module,
|
||
|
|
/// but leave its declaration.
|
||
|
|
void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
|
||
|
|
ValueToValueMapTy &VMap,
|
||
|
|
ValueMaterializer *Materializer = nullptr,
|
||
|
|
GlobalVariable *NewGV = nullptr);
|
||
|
|
|
||
|
|
/// Clone a global alias declaration into a new module.
|
||
|
|
GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
|
||
|
|
ValueToValueMapTy &VMap);
|
||
|
|
|
||
|
|
/// Clone module flags metadata into the destination module.
|
||
|
|
void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
|
||
|
|
ValueToValueMapTy &VMap);
|
||
|
|
|
||
|
|
} // end namespace orc
|
||
|
|
|
||
|
|
} // end namespace llvm
|
||
|
|
|
||
|
|
#endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
|