274 lines
9.5 KiB
C++
274 lines
9.5 KiB
C++
|
//===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==//
|
||
|
|
//
|
||
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
||
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||
|
|
//
|
||
|
|
//===----------------------------------------------------------------------===//
|
||
|
|
//
|
||
|
|
// This file implements the section-based memory manager used by the MCJIT
|
||
|
|
// execution engine and RuntimeDyld
|
||
|
|
//
|
||
|
|
//===----------------------------------------------------------------------===//
|
||
|
|
|
||
|
|
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
|
||
|
|
#include "llvm/Config/config.h"
|
||
|
|
#include "llvm/Support/MathExtras.h"
|
||
|
|
#include "llvm/Support/Process.h"
|
||
|
|
|
||
|
|
namespace llvm {
|
||
|
|
|
||
|
|
uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size,
|
||
|
|
unsigned Alignment,
|
||
|
|
unsigned SectionID,
|
||
|
|
StringRef SectionName,
|
||
|
|
bool IsReadOnly) {
|
||
|
|
if (IsReadOnly)
|
||
|
|
return allocateSection(SectionMemoryManager::AllocationPurpose::ROData,
|
||
|
|
Size, Alignment);
|
||
|
|
return allocateSection(SectionMemoryManager::AllocationPurpose::RWData, Size,
|
||
|
|
Alignment);
|
||
|
|
}
|
||
|
|
|
||
|
|
uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size,
|
||
|
|
unsigned Alignment,
|
||
|
|
unsigned SectionID,
|
||
|
|
StringRef SectionName) {
|
||
|
|
return allocateSection(SectionMemoryManager::AllocationPurpose::Code, Size,
|
||
|
|
Alignment);
|
||
|
|
}
|
||
|
|
|
||
|
|
uint8_t *SectionMemoryManager::allocateSection(
|
||
|
|
SectionMemoryManager::AllocationPurpose Purpose, uintptr_t Size,
|
||
|
|
unsigned Alignment) {
|
||
|
|
if (!Alignment)
|
||
|
|
Alignment = 16;
|
||
|
|
|
||
|
|
assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two.");
|
||
|
|
|
||
|
|
uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1) / Alignment + 1);
|
||
|
|
uintptr_t Addr = 0;
|
||
|
|
|
||
|
|
MemoryGroup &MemGroup = [&]() -> MemoryGroup & {
|
||
|
|
switch (Purpose) {
|
||
|
|
case AllocationPurpose::Code:
|
||
|
|
return CodeMem;
|
||
|
|
case AllocationPurpose::ROData:
|
||
|
|
return RODataMem;
|
||
|
|
case AllocationPurpose::RWData:
|
||
|
|
return RWDataMem;
|
||
|
|
}
|
||
|
|
llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose");
|
||
|
|
}();
|
||
|
|
|
||
|
|
// Look in the list of free memory regions and use a block there if one
|
||
|
|
// is available.
|
||
|
|
for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
|
||
|
|
if (FreeMB.Free.allocatedSize() >= RequiredSize) {
|
||
|
|
Addr = (uintptr_t)FreeMB.Free.base();
|
||
|
|
uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize();
|
||
|
|
// Align the address.
|
||
|
|
Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
|
||
|
|
|
||
|
|
if (FreeMB.PendingPrefixIndex == (unsigned)-1) {
|
||
|
|
// The part of the block we're giving out to the user is now pending
|
||
|
|
MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
|
||
|
|
|
||
|
|
// Remember this pending block, such that future allocations can just
|
||
|
|
// modify it rather than creating a new one
|
||
|
|
FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;
|
||
|
|
} else {
|
||
|
|
sys::MemoryBlock &PendingMB =
|
||
|
|
MemGroup.PendingMem[FreeMB.PendingPrefixIndex];
|
||
|
|
PendingMB = sys::MemoryBlock(PendingMB.base(),
|
||
|
|
Addr + Size - (uintptr_t)PendingMB.base());
|
||
|
|
}
|
||
|
|
|
||
|
|
// Remember how much free space is now left in this block
|
||
|
|
FreeMB.Free =
|
||
|
|
sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
|
||
|
|
return (uint8_t *)Addr;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// No pre-allocated free block was large enough. Allocate a new memory region.
|
||
|
|
// Note that all sections get allocated as read-write. The permissions will
|
||
|
|
// be updated later based on memory group.
|
||
|
|
//
|
||
|
|
// FIXME: It would be useful to define a default allocation size (or add
|
||
|
|
// it as a constructor parameter) to minimize the number of allocations.
|
||
|
|
//
|
||
|
|
// FIXME: Initialize the Near member for each memory group to avoid
|
||
|
|
// interleaving.
|
||
|
|
std::error_code ec;
|
||
|
|
sys::MemoryBlock MB = MMapper.allocateMappedMemory(
|
||
|
|
Purpose, RequiredSize, &MemGroup.Near,
|
||
|
|
sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec);
|
||
|
|
if (ec) {
|
||
|
|
// FIXME: Add error propagation to the interface.
|
||
|
|
return nullptr;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Save this address as the basis for our next request
|
||
|
|
MemGroup.Near = MB;
|
||
|
|
|
||
|
|
// Copy the address to all the other groups, if they have not
|
||
|
|
// been initialized.
|
||
|
|
if (CodeMem.Near.base() == 0)
|
||
|
|
CodeMem.Near = MB;
|
||
|
|
if (RODataMem.Near.base() == 0)
|
||
|
|
RODataMem.Near = MB;
|
||
|
|
if (RWDataMem.Near.base() == 0)
|
||
|
|
RWDataMem.Near = MB;
|
||
|
|
|
||
|
|
// Remember that we allocated this memory
|
||
|
|
MemGroup.AllocatedMem.push_back(MB);
|
||
|
|
Addr = (uintptr_t)MB.base();
|
||
|
|
uintptr_t EndOfBlock = Addr + MB.allocatedSize();
|
||
|
|
|
||
|
|
// Align the address.
|
||
|
|
Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
|
||
|
|
|
||
|
|
// The part of the block we're giving out to the user is now pending
|
||
|
|
MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
|
||
|
|
|
||
|
|
// The allocateMappedMemory may allocate much more memory than we need. In
|
||
|
|
// this case, we store the unused memory as a free memory block.
|
||
|
|
unsigned FreeSize = EndOfBlock - Addr - Size;
|
||
|
|
if (FreeSize > 16) {
|
||
|
|
FreeMemBlock FreeMB;
|
||
|
|
FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize);
|
||
|
|
FreeMB.PendingPrefixIndex = (unsigned)-1;
|
||
|
|
MemGroup.FreeMem.push_back(FreeMB);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Return aligned address
|
||
|
|
return (uint8_t *)Addr;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) {
|
||
|
|
// FIXME: Should in-progress permissions be reverted if an error occurs?
|
||
|
|
std::error_code ec;
|
||
|
|
|
||
|
|
// Make code memory executable.
|
||
|
|
ec = applyMemoryGroupPermissions(CodeMem,
|
||
|
|
sys::Memory::MF_READ | sys::Memory::MF_EXEC);
|
||
|
|
if (ec) {
|
||
|
|
if (ErrMsg) {
|
||
|
|
*ErrMsg = ec.message();
|
||
|
|
}
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Make read-only data memory read-only.
|
||
|
|
ec = applyMemoryGroupPermissions(RODataMem, sys::Memory::MF_READ);
|
||
|
|
if (ec) {
|
||
|
|
if (ErrMsg) {
|
||
|
|
*ErrMsg = ec.message();
|
||
|
|
}
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Read-write data memory already has the correct permissions
|
||
|
|
|
||
|
|
// Some platforms with separate data cache and instruction cache require
|
||
|
|
// explicit cache flush, otherwise JIT code manipulations (like resolved
|
||
|
|
// relocations) will get to the data cache but not to the instruction cache.
|
||
|
|
invalidateInstructionCache();
|
||
|
|
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) {
|
||
|
|
static const size_t PageSize = sys::Process::getPageSizeEstimate();
|
||
|
|
|
||
|
|
size_t StartOverlap =
|
||
|
|
(PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize;
|
||
|
|
|
||
|
|
size_t TrimmedSize = M.allocatedSize();
|
||
|
|
TrimmedSize -= StartOverlap;
|
||
|
|
TrimmedSize -= TrimmedSize % PageSize;
|
||
|
|
|
||
|
|
sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap),
|
||
|
|
TrimmedSize);
|
||
|
|
|
||
|
|
assert(((uintptr_t)Trimmed.base() % PageSize) == 0);
|
||
|
|
assert((Trimmed.allocatedSize() % PageSize) == 0);
|
||
|
|
assert(M.base() <= Trimmed.base() &&
|
||
|
|
Trimmed.allocatedSize() <= M.allocatedSize());
|
||
|
|
|
||
|
|
return Trimmed;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::error_code
|
||
|
|
SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,
|
||
|
|
unsigned Permissions) {
|
||
|
|
for (sys::MemoryBlock &MB : MemGroup.PendingMem)
|
||
|
|
if (std::error_code EC = MMapper.protectMappedMemory(MB, Permissions))
|
||
|
|
return EC;
|
||
|
|
|
||
|
|
MemGroup.PendingMem.clear();
|
||
|
|
|
||
|
|
// Now go through free blocks and trim any of them that don't span the entire
|
||
|
|
// page because one of the pending blocks may have overlapped it.
|
||
|
|
for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
|
||
|
|
FreeMB.Free = trimBlockToPageSize(FreeMB.Free);
|
||
|
|
// We cleared the PendingMem list, so all these pointers are now invalid
|
||
|
|
FreeMB.PendingPrefixIndex = (unsigned)-1;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Remove all blocks which are now empty
|
||
|
|
erase_if(MemGroup.FreeMem, [](FreeMemBlock &FreeMB) {
|
||
|
|
return FreeMB.Free.allocatedSize() == 0;
|
||
|
|
});
|
||
|
|
|
||
|
|
return std::error_code();
|
||
|
|
}
|
||
|
|
|
||
|
|
void SectionMemoryManager::invalidateInstructionCache() {
|
||
|
|
for (sys::MemoryBlock &Block : CodeMem.PendingMem)
|
||
|
|
sys::Memory::InvalidateInstructionCache(Block.base(),
|
||
|
|
Block.allocatedSize());
|
||
|
|
}
|
||
|
|
|
||
|
|
SectionMemoryManager::~SectionMemoryManager() {
|
||
|
|
for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) {
|
||
|
|
for (sys::MemoryBlock &Block : Group->AllocatedMem)
|
||
|
|
MMapper.releaseMappedMemory(Block);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
SectionMemoryManager::MemoryMapper::~MemoryMapper() {}
|
||
|
|
|
||
|
|
void SectionMemoryManager::anchor() {}
|
||
|
|
|
||
|
|
namespace {
|
||
|
|
// Trivial implementation of SectionMemoryManager::MemoryMapper that just calls
|
||
|
|
// into sys::Memory.
|
||
|
|
class DefaultMMapper final : public SectionMemoryManager::MemoryMapper {
|
||
|
|
public:
|
||
|
|
sys::MemoryBlock
|
||
|
|
allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose,
|
||
|
|
size_t NumBytes, const sys::MemoryBlock *const NearBlock,
|
||
|
|
unsigned Flags, std::error_code &EC) override {
|
||
|
|
return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, EC);
|
||
|
|
}
|
||
|
|
|
||
|
|
std::error_code protectMappedMemory(const sys::MemoryBlock &Block,
|
||
|
|
unsigned Flags) override {
|
||
|
|
return sys::Memory::protectMappedMemory(Block, Flags);
|
||
|
|
}
|
||
|
|
|
||
|
|
std::error_code releaseMappedMemory(sys::MemoryBlock &M) override {
|
||
|
|
return sys::Memory::releaseMappedMemory(M);
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
DefaultMMapper DefaultMMapperInstance;
|
||
|
|
} // namespace
|
||
|
|
|
||
|
|
SectionMemoryManager::SectionMemoryManager(MemoryMapper *MM)
|
||
|
|
: MMapper(MM ? *MM : DefaultMMapperInstance) {}
|
||
|
|
|
||
|
|
} // namespace llvm
|