2015-04-20 22:41:45 +02:00
|
|
|
//===------ OrcTestCommon.h - Utilities for Orc Unit Tests ------*- C++ -*-===//
|
|
|
|
//
|
2019-01-19 09:50:56 +01:00
|
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
2015-04-20 22:41:45 +02:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// Common utilities for the Orc unit tests.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
|
|
|
|
#ifndef LLVM_UNITTESTS_EXECUTIONENGINE_ORC_ORCTESTCOMMON_H
|
|
|
|
#define LLVM_UNITTESTS_EXECUTIONENGINE_ORC_ORCTESTCOMMON_H
|
|
|
|
|
2017-06-06 13:06:56 +02:00
|
|
|
#include "llvm/ExecutionEngine/JITSymbol.h"
|
2021-07-27 05:50:19 +02:00
|
|
|
#include "llvm/ExecutionEngine/Orc/Core.h"
|
|
|
|
#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
|
2018-03-28 05:41:45 +02:00
|
|
|
#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
|
2015-04-20 22:41:45 +02:00
|
|
|
#include "llvm/IR/Function.h"
|
|
|
|
#include "llvm/IR/IRBuilder.h"
|
|
|
|
#include "llvm/IR/LLVMContext.h"
|
|
|
|
#include "llvm/IR/Module.h"
|
2016-01-09 20:50:40 +01:00
|
|
|
#include "llvm/Object/ObjectFile.h"
|
2018-03-28 05:41:45 +02:00
|
|
|
#include "llvm/Support/TargetRegistry.h"
|
2018-07-20 20:31:50 +02:00
|
|
|
#include "llvm/Support/TargetSelect.h"
|
|
|
|
#include "gtest/gtest.h"
|
|
|
|
|
2015-04-20 22:41:45 +02:00
|
|
|
#include <memory>
|
|
|
|
|
|
|
|
namespace llvm {
|
|
|
|
|
2018-07-20 20:31:50 +02:00
|
|
|
namespace orc {
|
|
|
|
// CoreAPIsStandardTest that saves a bunch of boilerplate by providing the
|
|
|
|
// following:
|
|
|
|
//
|
|
|
|
// (1) ES -- An ExecutionSession
|
|
|
|
// (2) Foo, Bar, Baz, Qux -- SymbolStringPtrs for strings "foo", "bar", "baz",
|
|
|
|
// and "qux" respectively.
|
|
|
|
// (3) FooAddr, BarAddr, BazAddr, QuxAddr -- Dummy addresses. Guaranteed
|
|
|
|
// distinct and non-null.
|
|
|
|
// (4) FooSym, BarSym, BazSym, QuxSym -- JITEvaluatedSymbols with FooAddr,
|
|
|
|
// BarAddr, BazAddr, and QuxAddr respectively. All with default strong,
|
|
|
|
// linkage and non-hidden visibility.
|
2018-08-17 23:18:18 +02:00
|
|
|
// (5) V -- A JITDylib associated with ES.
|
2018-07-20 20:31:50 +02:00
|
|
|
class CoreAPIsBasedStandardTest : public testing::Test {
|
[ORC] Add support for resource tracking/removal (removable code).
This patch introduces new APIs to support resource tracking and removal in Orc.
It is intended as a thread-safe generalization of the removeModule concept from
OrcV1.
Clients can now create ResourceTracker objects (using
JITDylib::createResourceTracker) to track resources for each MaterializationUnit
(code, data, aliases, absolute symbols, etc.) added to the JIT. Every
MaterializationUnit will be associated with a ResourceTracker, and
ResourceTrackers can be re-used for multiple MaterializationUnits. Each JITDylib
has a default ResourceTracker that will be used for MaterializationUnits added
to that JITDylib if no ResourceTracker is explicitly specified.
Two operations can be performed on ResourceTrackers: transferTo and remove. The
transferTo operation transfers tracking of the resources to a different
ResourceTracker object, allowing ResourceTrackers to be merged to reduce
administrative overhead (the source tracker is invalidated in the process). The
remove operation removes all resources associated with a ResourceTracker,
including any symbols defined by MaterializationUnits associated with the
tracker, and also invalidates the tracker. These operations are thread safe, and
should work regardless of the the state of the MaterializationUnits. In the case
of resource transfer any existing resources associated with the source tracker
will be transferred to the destination tracker, and all future resources for
those units will be automatically associated with the destination tracker. In
the case of resource removal all already-allocated resources will be
deallocated, any if any program representations associated with the tracker have
not been compiled yet they will be destroyed. If any program representations are
currently being compiled then they will be prevented from completing: their
MaterializationResponsibility will return errors on any attempt to update the
JIT state.
Clients (usually Layer writers) wishing to track resources can implement the
ResourceManager API to receive notifications when ResourceTrackers are
transferred or removed. The MaterializationResponsibility::withResourceKeyDo
method can be used to create associations between the key for a ResourceTracker
and an allocated resource in a thread-safe way.
RTDyldObjectLinkingLayer and ObjectLinkingLayer are updated to use the
ResourceManager API to enable tracking and removal of memory allocated by the
JIT linker.
The new JITDylib::clear method can be used to trigger removal of every
ResourceTracker associated with the JITDylib (note that this will only
remove resources for the JITDylib, it does not run static destructors).
This patch includes unit tests showing basic usage. A follow-up patch will
update the Kaleidoscope and BuildingAJIT tutorial series to OrcV2 and will
use this API to release code associated with anonymous expressions.
2020-09-11 18:50:41 +02:00
|
|
|
public:
|
|
|
|
~CoreAPIsBasedStandardTest() {
|
|
|
|
if (auto Err = ES.endSession())
|
|
|
|
ES.reportError(std::move(Err));
|
|
|
|
}
|
|
|
|
|
2018-07-20 20:31:50 +02:00
|
|
|
protected:
|
2018-09-12 23:48:59 +02:00
|
|
|
std::shared_ptr<SymbolStringPool> SSP = std::make_shared<SymbolStringPool>();
|
2021-07-27 05:50:19 +02:00
|
|
|
ExecutionSession ES{std::make_unique<UnsupportedExecutorProcessControl>(SSP)};
|
[ORC] Add generic initializer/deinitializer support.
Initializers and deinitializers are used to implement C++ static constructors
and destructors, runtime registration for some languages (e.g. with the
Objective-C runtime for Objective-C/C++ code) and other tasks that would
typically be performed when a shared-object/dylib is loaded or unloaded by a
statically compiled program.
MCJIT and ORC have historically provided limited support for discovering and
running initializers/deinitializers by scanning the llvm.global_ctors and
llvm.global_dtors variables and recording the functions to be run. This approach
suffers from several drawbacks: (1) It only works for IR inputs, not for object
files (including cached JIT'd objects). (2) It only works for initializers
described by llvm.global_ctors and llvm.global_dtors, however not all
initializers are described in this way (Objective-C, for example, describes
initializers via specially named metadata sections). (3) To make the
initializer/deinitializer functions described by llvm.global_ctors and
llvm.global_dtors searchable they must be promoted to extern linkage, polluting
the JIT symbol table (extra care must be taken to ensure this promotion does
not result in symbol name clashes).
This patch introduces several interdependent changes to ORCv2 to support the
construction of new initialization schemes, and includes an implementation of a
backwards-compatible llvm.global_ctor/llvm.global_dtor scanning scheme, and a
MachO specific scheme that handles Objective-C runtime registration (if the
Objective-C runtime is available) enabling execution of LLVM IR compiled from
Objective-C and Swift.
The major changes included in this patch are:
(1) The MaterializationUnit and MaterializationResponsibility classes are
extended to describe an optional "initializer" symbol for the module (see the
getInitializerSymbol method on each class). The presence or absence of this
symbol indicates whether the module contains any initializers or
deinitializers. The initializer symbol otherwise behaves like any other:
searching for it triggers materialization.
(2) A new Platform interface is introduced in llvm/ExecutionEngine/Orc/Core.h
which provides the following callback interface:
- Error setupJITDylib(JITDylib &JD): Can be used to install standard symbols
in JITDylibs upon creation. E.g. __dso_handle.
- Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU): Generally
used to record initializer symbols.
- Error notifyRemoving(JITDylib &JD, VModuleKey K): Used to notify a platform
that a module is being removed.
Platform implementations can use these callbacks to track outstanding
initializers and implement a platform-specific approach for executing them. For
example, the MachOPlatform installs a plugin in the JIT linker to scan for both
__mod_inits sections (for C++ static constructors) and ObjC metadata sections.
If discovered, these are processed in the usual platform order: Objective-C
registration is carried out first, then static initializers are executed,
ensuring that calls to Objective-C from static initializers will be safe.
This patch updates LLJIT to use the new scheme for initialization. Two
LLJIT::PlatformSupport classes are implemented: A GenericIR platform and a MachO
platform. The GenericIR platform implements a modified version of the previous
llvm.global-ctor scraping scheme to provide support for Windows and
Linux. LLJIT's MachO platform uses the MachOPlatform class to provide MachO
specific initialization as described above.
Reviewers: sgraenitz, dblaikie
Subscribers: mgorny, hiraditya, mgrang, ributzka, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D74300
2019-12-16 11:50:40 +01:00
|
|
|
JITDylib &JD = ES.createBareJITDylib("JD");
|
2018-10-01 01:18:24 +02:00
|
|
|
SymbolStringPtr Foo = ES.intern("foo");
|
|
|
|
SymbolStringPtr Bar = ES.intern("bar");
|
|
|
|
SymbolStringPtr Baz = ES.intern("baz");
|
|
|
|
SymbolStringPtr Qux = ES.intern("qux");
|
2018-07-20 20:31:50 +02:00
|
|
|
static const JITTargetAddress FooAddr = 1U;
|
|
|
|
static const JITTargetAddress BarAddr = 2U;
|
|
|
|
static const JITTargetAddress BazAddr = 3U;
|
|
|
|
static const JITTargetAddress QuxAddr = 4U;
|
|
|
|
JITEvaluatedSymbol FooSym =
|
|
|
|
JITEvaluatedSymbol(FooAddr, JITSymbolFlags::Exported);
|
|
|
|
JITEvaluatedSymbol BarSym =
|
|
|
|
JITEvaluatedSymbol(BarAddr, JITSymbolFlags::Exported);
|
|
|
|
JITEvaluatedSymbol BazSym =
|
|
|
|
JITEvaluatedSymbol(BazAddr, JITSymbolFlags::Exported);
|
|
|
|
JITEvaluatedSymbol QuxSym =
|
|
|
|
JITEvaluatedSymbol(QuxAddr, JITSymbolFlags::Exported);
|
|
|
|
};
|
|
|
|
|
|
|
|
} // end namespace orc
|
|
|
|
|
2017-04-04 19:03:49 +02:00
|
|
|
class OrcNativeTarget {
|
2015-10-28 03:40:04 +01:00
|
|
|
public:
|
2017-04-04 19:03:49 +02:00
|
|
|
static void initialize() {
|
2015-10-28 03:40:04 +01:00
|
|
|
if (!NativeTargetInitialized) {
|
|
|
|
InitializeNativeTarget();
|
|
|
|
InitializeNativeTargetAsmParser();
|
|
|
|
InitializeNativeTargetAsmPrinter();
|
|
|
|
NativeTargetInitialized = true;
|
|
|
|
}
|
2017-04-04 19:03:49 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
static bool NativeTargetInitialized;
|
|
|
|
};
|
|
|
|
|
2018-09-26 06:18:30 +02:00
|
|
|
class SimpleMaterializationUnit : public orc::MaterializationUnit {
|
|
|
|
public:
|
|
|
|
using MaterializeFunction =
|
2020-09-11 18:23:14 +02:00
|
|
|
std::function<void(std::unique_ptr<orc::MaterializationResponsibility>)>;
|
2018-09-26 06:18:30 +02:00
|
|
|
using DiscardFunction =
|
|
|
|
std::function<void(const orc::JITDylib &, orc::SymbolStringPtr)>;
|
|
|
|
using DestructorFunction = std::function<void()>;
|
|
|
|
|
|
|
|
SimpleMaterializationUnit(
|
|
|
|
orc::SymbolFlagsMap SymbolFlags, MaterializeFunction Materialize,
|
[ORC] Add generic initializer/deinitializer support.
Initializers and deinitializers are used to implement C++ static constructors
and destructors, runtime registration for some languages (e.g. with the
Objective-C runtime for Objective-C/C++ code) and other tasks that would
typically be performed when a shared-object/dylib is loaded or unloaded by a
statically compiled program.
MCJIT and ORC have historically provided limited support for discovering and
running initializers/deinitializers by scanning the llvm.global_ctors and
llvm.global_dtors variables and recording the functions to be run. This approach
suffers from several drawbacks: (1) It only works for IR inputs, not for object
files (including cached JIT'd objects). (2) It only works for initializers
described by llvm.global_ctors and llvm.global_dtors, however not all
initializers are described in this way (Objective-C, for example, describes
initializers via specially named metadata sections). (3) To make the
initializer/deinitializer functions described by llvm.global_ctors and
llvm.global_dtors searchable they must be promoted to extern linkage, polluting
the JIT symbol table (extra care must be taken to ensure this promotion does
not result in symbol name clashes).
This patch introduces several interdependent changes to ORCv2 to support the
construction of new initialization schemes, and includes an implementation of a
backwards-compatible llvm.global_ctor/llvm.global_dtor scanning scheme, and a
MachO specific scheme that handles Objective-C runtime registration (if the
Objective-C runtime is available) enabling execution of LLVM IR compiled from
Objective-C and Swift.
The major changes included in this patch are:
(1) The MaterializationUnit and MaterializationResponsibility classes are
extended to describe an optional "initializer" symbol for the module (see the
getInitializerSymbol method on each class). The presence or absence of this
symbol indicates whether the module contains any initializers or
deinitializers. The initializer symbol otherwise behaves like any other:
searching for it triggers materialization.
(2) A new Platform interface is introduced in llvm/ExecutionEngine/Orc/Core.h
which provides the following callback interface:
- Error setupJITDylib(JITDylib &JD): Can be used to install standard symbols
in JITDylibs upon creation. E.g. __dso_handle.
- Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU): Generally
used to record initializer symbols.
- Error notifyRemoving(JITDylib &JD, VModuleKey K): Used to notify a platform
that a module is being removed.
Platform implementations can use these callbacks to track outstanding
initializers and implement a platform-specific approach for executing them. For
example, the MachOPlatform installs a plugin in the JIT linker to scan for both
__mod_inits sections (for C++ static constructors) and ObjC metadata sections.
If discovered, these are processed in the usual platform order: Objective-C
registration is carried out first, then static initializers are executed,
ensuring that calls to Objective-C from static initializers will be safe.
This patch updates LLJIT to use the new scheme for initialization. Two
LLJIT::PlatformSupport classes are implemented: A GenericIR platform and a MachO
platform. The GenericIR platform implements a modified version of the previous
llvm.global-ctor scraping scheme to provide support for Windows and
Linux. LLJIT's MachO platform uses the MachOPlatform class to provide MachO
specific initialization as described above.
Reviewers: sgraenitz, dblaikie
Subscribers: mgorny, hiraditya, mgrang, ributzka, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D74300
2019-12-16 11:50:40 +01:00
|
|
|
orc::SymbolStringPtr InitSym = nullptr,
|
2018-09-26 06:18:30 +02:00
|
|
|
DiscardFunction Discard = DiscardFunction(),
|
|
|
|
DestructorFunction Destructor = DestructorFunction())
|
[ORC] Add support for resource tracking/removal (removable code).
This patch introduces new APIs to support resource tracking and removal in Orc.
It is intended as a thread-safe generalization of the removeModule concept from
OrcV1.
Clients can now create ResourceTracker objects (using
JITDylib::createResourceTracker) to track resources for each MaterializationUnit
(code, data, aliases, absolute symbols, etc.) added to the JIT. Every
MaterializationUnit will be associated with a ResourceTracker, and
ResourceTrackers can be re-used for multiple MaterializationUnits. Each JITDylib
has a default ResourceTracker that will be used for MaterializationUnits added
to that JITDylib if no ResourceTracker is explicitly specified.
Two operations can be performed on ResourceTrackers: transferTo and remove. The
transferTo operation transfers tracking of the resources to a different
ResourceTracker object, allowing ResourceTrackers to be merged to reduce
administrative overhead (the source tracker is invalidated in the process). The
remove operation removes all resources associated with a ResourceTracker,
including any symbols defined by MaterializationUnits associated with the
tracker, and also invalidates the tracker. These operations are thread safe, and
should work regardless of the the state of the MaterializationUnits. In the case
of resource transfer any existing resources associated with the source tracker
will be transferred to the destination tracker, and all future resources for
those units will be automatically associated with the destination tracker. In
the case of resource removal all already-allocated resources will be
deallocated, any if any program representations associated with the tracker have
not been compiled yet they will be destroyed. If any program representations are
currently being compiled then they will be prevented from completing: their
MaterializationResponsibility will return errors on any attempt to update the
JIT state.
Clients (usually Layer writers) wishing to track resources can implement the
ResourceManager API to receive notifications when ResourceTrackers are
transferred or removed. The MaterializationResponsibility::withResourceKeyDo
method can be used to create associations between the key for a ResourceTracker
and an allocated resource in a thread-safe way.
RTDyldObjectLinkingLayer and ObjectLinkingLayer are updated to use the
ResourceManager API to enable tracking and removal of memory allocated by the
JIT linker.
The new JITDylib::clear method can be used to trigger removal of every
ResourceTracker associated with the JITDylib (note that this will only
remove resources for the JITDylib, it does not run static destructors).
This patch includes unit tests showing basic usage. A follow-up patch will
update the Kaleidoscope and BuildingAJIT tutorial series to OrcV2 and will
use this API to release code associated with anonymous expressions.
2020-09-11 18:50:41 +02:00
|
|
|
: MaterializationUnit(std::move(SymbolFlags), std::move(InitSym)),
|
2018-09-26 06:18:30 +02:00
|
|
|
Materialize(std::move(Materialize)), Discard(std::move(Discard)),
|
|
|
|
Destructor(std::move(Destructor)) {}
|
|
|
|
|
|
|
|
~SimpleMaterializationUnit() override {
|
|
|
|
if (Destructor)
|
|
|
|
Destructor();
|
|
|
|
}
|
|
|
|
|
2018-09-28 17:09:14 +02:00
|
|
|
StringRef getName() const override { return "<Simple>"; }
|
|
|
|
|
2020-09-11 18:23:14 +02:00
|
|
|
void
|
|
|
|
materialize(std::unique_ptr<orc::MaterializationResponsibility> R) override {
|
2018-09-26 06:18:30 +02:00
|
|
|
Materialize(std::move(R));
|
|
|
|
}
|
|
|
|
|
2018-10-07 01:02:06 +02:00
|
|
|
void discard(const orc::JITDylib &JD,
|
|
|
|
const orc::SymbolStringPtr &Name) override {
|
2018-09-26 06:18:30 +02:00
|
|
|
if (Discard)
|
|
|
|
Discard(JD, std::move(Name));
|
|
|
|
else
|
|
|
|
llvm_unreachable("Discard not supported");
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
MaterializeFunction Materialize;
|
|
|
|
DiscardFunction Discard;
|
|
|
|
DestructorFunction Destructor;
|
|
|
|
};
|
|
|
|
|
2015-10-27 18:45:48 +01:00
|
|
|
class ModuleBuilder {
|
|
|
|
public:
|
|
|
|
ModuleBuilder(LLVMContext &Context, StringRef Triple,
|
|
|
|
StringRef Name);
|
2015-04-20 22:41:45 +02:00
|
|
|
|
2019-01-13 17:09:28 +01:00
|
|
|
Function *createFunctionDecl(FunctionType *FTy, StringRef Name) {
|
|
|
|
return Function::Create(FTy, GlobalValue::ExternalLinkage, Name, M.get());
|
2015-10-27 18:45:48 +01:00
|
|
|
}
|
2015-04-20 22:41:45 +02:00
|
|
|
|
2015-10-27 18:45:48 +01:00
|
|
|
Module* getModule() { return M.get(); }
|
|
|
|
const Module* getModule() const { return M.get(); }
|
|
|
|
std::unique_ptr<Module> takeModule() { return std::move(M); }
|
2015-04-20 22:41:45 +02:00
|
|
|
|
2015-10-27 18:45:48 +01:00
|
|
|
private:
|
|
|
|
std::unique_ptr<Module> M;
|
|
|
|
};
|
2015-04-20 22:41:45 +02:00
|
|
|
|
2015-10-27 18:45:48 +01:00
|
|
|
// Dummy struct type.
|
|
|
|
struct DummyStruct {
|
|
|
|
int X[256];
|
|
|
|
};
|
2015-04-20 22:41:45 +02:00
|
|
|
|
2019-01-13 17:09:28 +01:00
|
|
|
inline StructType *getDummyStructTy(LLVMContext &Context) {
|
|
|
|
return StructType::get(ArrayType::get(Type::getInt32Ty(Context), 256));
|
|
|
|
}
|
2015-04-20 22:41:45 +02:00
|
|
|
|
|
|
|
} // namespace llvm
|
|
|
|
|
|
|
|
#endif
|