1
0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-11-22 18:54:02 +01:00

Add support for archives and object file caching under MCJIT.

Patch by Andy Kaylor, with minor edits to resolve merge conflicts.

llvm-svn: 196639
This commit is contained in:
Lang Hames 2013-12-07 03:05:51 +00:00
parent db406f2a95
commit a8f3295449
13 changed files with 342 additions and 10 deletions

View File

@ -48,6 +48,11 @@ class RTDyldMemoryManager;
class Triple;
class Type;
namespace object {
class Archive;
class ObjectFile;
}
/// \brief Helper class for helping synchronize access to the global address map
/// table.
class ExecutionEngineState {
@ -204,6 +209,33 @@ public:
Modules.push_back(M);
}
/// addObjectFile - Add an ObjectFile to the execution engine.
///
/// This method is only supported by MCJIT. MCJIT will immediately load the
/// object into memory and adds its symbols to the list used to resolve
/// external symbols while preparing other objects for execution.
///
/// Objects added using this function will not be made executable until
/// needed by another object.
///
/// MCJIT will take ownership of the ObjectFile.
virtual void addObjectFile(object::ObjectFile *O) {
llvm_unreachable(
"ExecutionEngine subclass doesn't implement addObjectFile.");
}
/// addArchive - Add an Archive to the execution engine.
///
/// This method is only supported by MCJIT. MCJIT will use the archive to
/// resolve external symbols in objects it is loading. If a symbol is found
/// in the Archive the contained object file will be extracted (in memory)
/// and loaded for possible execution.
///
/// MCJIT will take ownership of the Archive.
virtual void addArchive(object::Archive *A) {
llvm_unreachable("ExecutionEngine subclass doesn't implement addArchive.");
}
//===--------------------------------------------------------------------===//
const DataLayout *getDataLayout() const { return TD; }

View File

@ -21,6 +21,10 @@
namespace llvm {
namespace object {
class ObjectFile;
}
class RuntimeDyldImpl;
class ObjectImage;
@ -46,6 +50,12 @@ public:
/// failure, the input buffer will be deleted.
ObjectImage *loadObject(ObjectBuffer *InputBuffer);
/// Prepare the referenced object file for execution.
/// Ownership of the input object is transferred to the ObjectImage
/// instance returned from this function if successful. In the case of load
/// failure, the input object will be deleted.
ObjectImage *loadObject(object::ObjectFile *InputObject);
/// Get the address of our local copy of the symbol. This may or may not
/// be the address used for relocation (clients can copy the data around
/// and resolve relocatons based on where they put it).

View File

@ -21,6 +21,7 @@
#include "llvm/IR/Function.h"
#include "llvm/IR/Module.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/Object/Archive.h"
#include "llvm/Support/DynamicLibrary.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MemoryBuffer.h"
@ -78,15 +79,24 @@ MCJIT::~MCJIT() {
Modules.clear();
Dyld.deregisterEHFrames();
LoadedObjectMap::iterator it, end = LoadedObjects.end();
for (it = LoadedObjects.begin(); it != end; ++it) {
ObjectImage *Obj = it->second;
LoadedObjectList::iterator it, end;
for (it = LoadedObjects.begin(), end = LoadedObjects.end(); it != end; ++it) {
ObjectImage *Obj = *it;
if (Obj) {
NotifyFreeingObject(*Obj);
delete Obj;
}
}
LoadedObjects.clear();
SmallVector<object::Archive *, 2>::iterator ArIt, ArEnd;
for (ArIt = Archives.begin(), ArEnd = Archives.end(); ArIt != ArEnd; ++ArIt) {
object::Archive *A = *ArIt;
delete A;
}
Archives.clear();
delete TM;
}
@ -102,6 +112,21 @@ bool MCJIT::removeModule(Module *M) {
void MCJIT::addObjectFile(object::ObjectFile *Obj) {
ObjectImage *LoadedObject = Dyld.loadObject(Obj);
if (!LoadedObject)
report_fatal_error(Dyld.getErrorString());
LoadedObjects.push_back(LoadedObject);
NotifyObjectEmitted(*LoadedObject);
}
void MCJIT::addArchive(object::Archive *A) {
Archives.push_back(A);
}
void MCJIT::setObjectCache(ObjectCache* NewCache) {
MutexGuard locked(lock);
ObjCache = NewCache;
@ -171,9 +196,9 @@ void MCJIT::generateCodeForModule(Module *M) {
}
// Load the object into the dynamic linker.
// MCJIT now owns the ObjectImage pointer (via its LoadedObjects map).
// MCJIT now owns the ObjectImage pointer (via its LoadedObjects list).
ObjectImage *LoadedObject = Dyld.loadObject(ObjectToLoad.take());
LoadedObjects[M] = LoadedObject;
LoadedObjects.push_back(LoadedObject);
if (!LoadedObject)
report_fatal_error(Dyld.getErrorString());
@ -271,6 +296,27 @@ uint64_t MCJIT::getSymbolAddress(const std::string &Name,
if (Addr)
return Addr;
SmallVector<object::Archive*, 2>::iterator I, E;
for (I = Archives.begin(), E = Archives.end(); I != E; ++I) {
object::Archive *A = *I;
// Look for our symbols in each Archive
object::Archive::child_iterator ChildIt = A->findSym(Name);
if (ChildIt != A->end_children()) {
OwningPtr<object::Binary> ChildBin;
// FIXME: Support nested archives?
if (!ChildIt->getAsBinary(ChildBin) && ChildBin->isObject()) {
object::ObjectFile *OF = reinterpret_cast<object::ObjectFile *>(
ChildBin.take());
// This causes the object file to be loaded.
addObjectFile(OF);
// The address should be here now.
Addr = getExistingSymbolAddress(Name);
if (Addr)
return Addr;
}
}
}
// If it hasn't already been generated, see if it's in one of our modules.
Module *M = findModuleForSymbol(Name, CheckFunctionsOnly);
if (!M)

View File

@ -206,8 +206,10 @@ class MCJIT : public ExecutionEngine {
OwningModuleContainer OwnedModules;
typedef DenseMap<Module *, ObjectImage *> LoadedObjectMap;
LoadedObjectMap LoadedObjects;
SmallVector<object::Archive*, 2> Archives;
typedef SmallVector<ObjectImage *, 2> LoadedObjectList;
LoadedObjectList LoadedObjects;
// An optional ObjectCache to be notified of compiled objects and used to
// perform lookup of pre-compiled code to avoid re-compilation.
@ -227,6 +229,8 @@ public:
/// @name ExecutionEngine interface implementation
/// @{
virtual void addModule(Module *M);
virtual void addObjectFile(object::ObjectFile *O);
virtual void addArchive(object::Archive *O);
virtual bool removeModule(Module *M);
/// FindFunctionNamed - Search all of the active modules to find the one that

View File

@ -20,6 +20,10 @@
namespace llvm {
namespace object {
class ObjectFile;
}
class ObjectImageCommon : public ObjectImage {
ObjectImageCommon(); // = delete
ObjectImageCommon(const ObjectImageCommon &other); // = delete
@ -42,6 +46,8 @@ public:
{
ObjFile = object::ObjectFile::createObjectFile(Buffer->getMemBuffer());
}
ObjectImageCommon(object::ObjectFile* Input)
: ObjectImage(NULL), ObjFile(Input) {}
virtual ~ObjectImageCommon() { delete ObjFile; }
virtual object::symbol_iterator begin_symbols() const

View File

@ -82,12 +82,24 @@ ObjectImage *RuntimeDyldImpl::createObjectImage(ObjectBuffer *InputBuffer) {
return new ObjectImageCommon(InputBuffer);
}
ObjectImage *RuntimeDyldImpl::createObjectImageFromFile(ObjectFile *InputObject) {
return new ObjectImageCommon(InputObject);
}
ObjectImage *RuntimeDyldImpl::loadObject(ObjectFile *InputObject) {
return loadObject(createObjectImageFromFile(InputObject));
}
ObjectImage *RuntimeDyldImpl::loadObject(ObjectBuffer *InputBuffer) {
return loadObject(createObjectImage(InputBuffer));
}
ObjectImage *RuntimeDyldImpl::loadObject(ObjectImage *InputObject) {
MutexGuard locked(lock);
OwningPtr<ObjectImage> obj(createObjectImage(InputBuffer));
OwningPtr<ObjectImage> obj(InputObject);
if (!obj)
report_fatal_error("Unable to create object image from memory buffer!");
return NULL;
// Save information about our target
Arch = (Triple::ArchType)obj->getArch();
@ -139,7 +151,7 @@ ObjectImage *RuntimeDyldImpl::loadObject(ObjectBuffer *InputBuffer) {
if (si == obj->end_sections()) continue;
Check(si->getContents(SectionData));
Check(si->isText(IsCode));
const uint8_t* SymPtr = (const uint8_t*)InputBuffer->getBufferStart() +
const uint8_t* SymPtr = (const uint8_t*)InputObject->getData().data() +
(uintptr_t)FileOffset;
uintptr_t SectOffset = (uintptr_t)(SymPtr -
(const uint8_t*)SectionData.begin());
@ -563,6 +575,22 @@ RuntimeDyld::~RuntimeDyld() {
delete Dyld;
}
ObjectImage *RuntimeDyld::loadObject(ObjectFile *InputObject) {
if (!Dyld) {
if (InputObject->isELF())
Dyld = new RuntimeDyldELF(MM);
else if (InputObject->isMachO())
Dyld = new RuntimeDyldMachO(MM);
else
report_fatal_error("Incompatible object format!");
} else {
if (!Dyld->isCompatibleFile(InputObject))
report_fatal_error("Incompatible object format!");
}
return Dyld->loadObject(InputObject);
}
ObjectImage *RuntimeDyld::loadObject(ObjectBuffer *InputBuffer) {
if (!Dyld) {
sys::fs::file_magic Type =

View File

@ -25,6 +25,8 @@
#include "llvm/Object/ELFObjectFile.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Support/ELF.h"
#include "llvm/Support/MemoryBuffer.h"
using namespace llvm;
using namespace llvm::object;
@ -178,6 +180,39 @@ void RuntimeDyldELF::deregisterEHFrames() {
RegisteredEHFrameSections.clear();
}
ObjectImage *RuntimeDyldELF::createObjectImageFromFile(object::ObjectFile *ObjFile) {
if (!ObjFile)
return NULL;
error_code ec;
MemoryBuffer* Buffer = MemoryBuffer::getMemBuffer(ObjFile->getData(),
"",
false);
if (ObjFile->getBytesInAddress() == 4 && ObjFile->isLittleEndian()) {
DyldELFObject<ELFType<support::little, 4, false> > *Obj =
new DyldELFObject<ELFType<support::little, 4, false> >(Buffer, ec);
return new ELFObjectImage<ELFType<support::little, 4, false> >(NULL, Obj);
}
else if (ObjFile->getBytesInAddress() == 4 && !ObjFile->isLittleEndian()) {
DyldELFObject<ELFType<support::big, 4, false> > *Obj =
new DyldELFObject<ELFType<support::big, 4, false> >(Buffer, ec);
return new ELFObjectImage<ELFType<support::big, 4, false> >(NULL, Obj);
}
else if (ObjFile->getBytesInAddress() == 8 && !ObjFile->isLittleEndian()) {
DyldELFObject<ELFType<support::big, 8, true> > *Obj =
new DyldELFObject<ELFType<support::big, 8, true> >(Buffer, ec);
return new ELFObjectImage<ELFType<support::big, 8, true> >(NULL, Obj);
}
else if (ObjFile->getBytesInAddress() == 8 && ObjFile->isLittleEndian()) {
DyldELFObject<ELFType<support::little, 8, true> > *Obj =
new DyldELFObject<ELFType<support::little, 8, true> >(Buffer, ec);
return new ELFObjectImage<ELFType<support::little, 8, true> >(NULL, Obj);
}
else
llvm_unreachable("Unexpected ELF format");
}
ObjectImage *RuntimeDyldELF::createObjectImage(ObjectBuffer *Buffer) {
if (Buffer->getBufferSize() < ELF::EI_NIDENT)
llvm_unreachable("Unexpected ELF object size");
@ -1403,4 +1438,9 @@ bool RuntimeDyldELF::isCompatibleFormat(const ObjectBuffer *Buffer) const {
return false;
return (memcmp(Buffer->getBufferStart(), ELF::ElfMagic, strlen(ELF::ElfMagic))) == 0;
}
bool RuntimeDyldELF::isCompatibleFile(const object::ObjectFile *Obj) const {
return Obj->isELF();
}
} // namespace llvm

View File

@ -140,7 +140,9 @@ public:
const SymbolTableMap &Symbols,
StubMap &Stubs);
virtual bool isCompatibleFormat(const ObjectBuffer *Buffer) const;
virtual bool isCompatibleFile(const object::ObjectFile *Buffer) const;
virtual ObjectImage *createObjectImage(ObjectBuffer *InputBuffer);
virtual ObjectImage *createObjectImageFromFile(object::ObjectFile *Obj);
virtual void registerEHFrames();
virtual void deregisterEHFrames();
virtual void finalizeLoad(ObjSectionToIDMap &SectionMap);

View File

@ -311,12 +311,18 @@ protected:
virtual void updateGOTEntries(StringRef Name, uint64_t Addr) {}
virtual ObjectImage *createObjectImage(ObjectBuffer *InputBuffer);
virtual ObjectImage *createObjectImageFromFile(object::ObjectFile *InputObject);
// This is the implementation for the two public overloads
ObjectImage *loadObject(ObjectImage *InputObject);
public:
RuntimeDyldImpl(RTDyldMemoryManager *mm) : MemMgr(mm), HasError(false) {}
virtual ~RuntimeDyldImpl();
ObjectImage *loadObject(ObjectBuffer *InputBuffer);
ObjectImage *loadObject(object::ObjectFile *InputObject);
void *getSymbolAddress(StringRef Name) {
// FIXME: Just look up as a function for now. Overly simple of course.
@ -354,6 +360,7 @@ public:
StringRef getErrorString() { return ErrorStr; }
virtual bool isCompatibleFormat(const ObjectBuffer *Buffer) const = 0;
virtual bool isCompatibleFile(const ObjectFile *Obj) const = 0;
virtual void registerEHFrames();

View File

@ -455,4 +455,9 @@ bool RuntimeDyldMachO::isCompatibleFormat(
return false;
}
bool RuntimeDyldMachO::isCompatibleFile(
const object::ObjectFile *Obj) const {
return Obj->isMachO();
}
} // end namespace llvm

View File

@ -94,6 +94,7 @@ public:
const SymbolTableMap &Symbols,
StubMap &Stubs);
virtual bool isCompatibleFormat(const ObjectBuffer *Buffer) const;
virtual bool isCompatibleFile(const object::ObjectFile *Obj) const;
virtual void registerEHFrames();
virtual void finalizeLoad(ObjSectionToIDMap &SectionMap);
};

View File

@ -0,0 +1,28 @@
; REQUIRES: shell
; This first line will generate the .o files for the next run line
; RUN: %lli_mcjit -extra-module=%p/Inputs/multi-module-b.ll -extra-module=%p/Inputs/multi-module-c.ll -enable-cache-manager %s
; This line tests MCJIT object loading
; RUN: %lli_mcjit -extra-object=%p/Inputs/multi-module-b.o -extra-object=%p/Inputs/multi-module-c.o %s
; These lines put the object files into an archive
; RUN: llvm-ar r %p/Inputs/load-object.a %p/Inputs/multi-module-b.o
; RUN: llvm-ar r %p/Inputs/load-object.a %p/Inputs/multi-module-c.o
; This line test MCJIT archive loading
; RUN: %lli_mcjit -extra-archive=%p/Inputs/load-object.a %s
; These lines clean up our temporary files
; RUN: rm -f %p/Inputs/load-object-a.o
; RUN: rm -f %p/Inputs/multi-module-b.o
; RUN: rm -f %p/Inputs/multi-module-c.o
; RUN: rm -f %p/Inputs/load-object.a
declare i32 @FB()
define i32 @main() {
%r = call i32 @FB( ) ; <i32> [#uses=1]
ret i32 %r
}

View File

@ -26,12 +26,15 @@
#include "llvm/ExecutionEngine/JITEventListener.h"
#include "llvm/ExecutionEngine/JITMemoryManager.h"
#include "llvm/ExecutionEngine/MCJIT.h"
#include "llvm/ExecutionEngine/ObjectCache.h"
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/TypeBuilder.h"
#include "llvm/IRReader/IRReader.h"
#include "llvm/Object/Archive.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DynamicLibrary.h"
@ -50,6 +53,7 @@
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Instrumentation.h"
#include <cerrno>
#include <fstream>
#ifdef __CYGWIN__
#include <cygwin/version.h>
@ -138,6 +142,21 @@ namespace {
cl::desc("Extra modules to be loaded"),
cl::value_desc("input bitcode"));
cl::list<std::string>
ExtraObjects("extra-object",
cl::desc("Extra object files to be loaded"),
cl::value_desc("input object"));
cl::list<std::string>
ExtraArchives("extra-archive",
cl::desc("Extra archive files to be loaded"),
cl::value_desc("input archive"));
cl::opt<bool>
EnableCacheManager("enable-cache-manager",
cl::desc("Use cache manager to save/load mdoules."),
cl::init(false));
cl::opt<std::string>
FakeArgv0("fake-argv0",
cl::desc("Override the 'argv[0]' value passed into the executing"
@ -219,12 +238,68 @@ namespace {
cl::init(false));
}
//===----------------------------------------------------------------------===//
// Object cache
//
// This object cache implementation writes cached objects to disk using a
// filename provided in the module descriptor and tries to load a saved object
// using that filename if the file exists.
//
class LLIObjectCache : public ObjectCache {
public:
LLIObjectCache() { }
virtual ~LLIObjectCache() {}
virtual void notifyObjectCompiled(const Module *M, const MemoryBuffer *Obj) {
const std::string ModuleID = M->getModuleIdentifier();
std::string CacheName;
if (!getCacheFilename(ModuleID, CacheName))
return;
std::ofstream outfile(CacheName.c_str(), std::ofstream::binary);
outfile.write(Obj->getBufferStart(), Obj->getBufferSize());
outfile.close();
}
virtual MemoryBuffer* getObject(const Module* M) {
const std::string ModuleID = M->getModuleIdentifier();
std::string CacheName;
if (!getCacheFilename(ModuleID, CacheName))
return NULL;
// Load the object from the cache filename
OwningPtr<MemoryBuffer> IRObjectBuffer;
MemoryBuffer::getFile(CacheName.c_str(), IRObjectBuffer, -1, false);
// If the file isn't there, that's OK.
if (!IRObjectBuffer)
return NULL;
// MCJIT will want to write into this buffer, and we don't want that
// because the file has probably just been mmapped. Instead we make
// a copy. The filed-based buffer will be released when it goes
// out of scope.
return MemoryBuffer::getMemBufferCopy(IRObjectBuffer->getBuffer());
}
private:
bool getCacheFilename(const std::string &ModID, std::string &CacheName) {
std::string Prefix("file:");
size_t PrefixLength = Prefix.length();
if (ModID.substr(0, PrefixLength) != Prefix)
return false;
CacheName = ModID.substr(PrefixLength);
size_t pos = CacheName.rfind('.');
CacheName.replace(pos, CacheName.length() - pos, ".o");
return true;
}
};
static ExecutionEngine *EE = 0;
static LLIObjectCache *CacheManager = 0;
static void do_shutdown() {
// Cygwin-1.5 invokes DLL's dtors before atexit handler.
#ifndef DO_NOTHING_ATEXIT
delete EE;
if (CacheManager)
delete CacheManager;
llvm_shutdown();
#endif
}
@ -300,6 +375,15 @@ int main(int argc, char **argv, char * const *envp) {
return 1;
}
if (EnableCacheManager) {
if (UseMCJIT) {
std::string CacheName("file:");
CacheName.append(InputFile);
Mod->setModuleIdentifier(CacheName);
} else
errs() << "warning: -enable-cache-manager can only be used with MCJIT.";
}
// If not jitting lazily, load the whole bitcode file eagerly too.
std::string ErrorMsg;
if (NoLazyCompilation) {
@ -391,6 +475,11 @@ int main(int argc, char **argv, char * const *envp) {
exit(1);
}
if (EnableCacheManager) {
CacheManager = new LLIObjectCache;
EE->setObjectCache(CacheManager);
}
// Load any additional modules specified on the command line.
for (unsigned i = 0, e = ExtraModules.size(); i != e; ++i) {
Module *XMod = ParseIRFile(ExtraModules[i], Err, Context);
@ -398,9 +487,43 @@ int main(int argc, char **argv, char * const *envp) {
Err.print(argv[0], errs());
return 1;
}
if (EnableCacheManager) {
if (UseMCJIT) {
std::string CacheName("file:");
CacheName.append(ExtraModules[i]);
XMod->setModuleIdentifier(CacheName);
}
// else, we already printed a warning above.
}
EE->addModule(XMod);
}
for (unsigned i = 0, e = ExtraObjects.size(); i != e; ++i) {
object::ObjectFile *Obj = object::ObjectFile::createObjectFile(
ExtraObjects[i]);
if (!Obj) {
Err.print(argv[0], errs());
return 1;
}
EE->addObjectFile(Obj);
}
for (unsigned i = 0, e = ExtraArchives.size(); i != e; ++i) {
OwningPtr<MemoryBuffer> ArBuf;
error_code ec;
ec = MemoryBuffer::getFileOrSTDIN(ExtraArchives[i], ArBuf);
if (ec) {
Err.print(argv[0], errs());
return 1;
}
object::Archive *Ar = new object::Archive(ArBuf.take(), ec);
if (ec || !Ar) {
Err.print(argv[0], errs());
return 1;
}
EE->addArchive(Ar);
}
// If the target is Cygwin/MingW and we are generating remote code, we
// need an extra module to help out with linking.
if (RemoteMCJIT && Triple(Mod->getTargetTriple()).isOSCygMing()) {