$(TARGET:%=$(ObjDir)/%GenMCCodeEmitter.inc.tmp): \
$(ObjDir)/%GenMCCodeEmitter.inc.tmp: %.td $(ObjDir)/.dir $(LLVM_TBLGEN)
$(Echo) "Building $(<F) MC code emitter with tblgen"
- $(Verb) $(LLVMTableGen) -gen-emitter -mc-emitter -o $(call SYSPATH, $@) $<
+ $(Verb) $(LLVMTableGen) -gen-emitter -o $(call SYSPATH, $@) $<
$(TARGET:%=$(ObjDir)/%GenMCPseudoLowering.inc.tmp): \
$(ObjDir)/%GenMCPseudoLowering.inc.tmp: %.td $(ObjDir)/.dir $(LLVM_TBLGEN)
$(Echo) "Building $(<F) MC Pseudo instruction expander with tblgen"
$(Verb) $(LLVMTableGen) -gen-pseudo-lowering -o $(call SYSPATH, $@) $<
-$(TARGET:%=$(ObjDir)/%GenCodeEmitter.inc.tmp): \
-$(ObjDir)/%GenCodeEmitter.inc.tmp: %.td $(ObjDir)/.dir $(LLVM_TBLGEN)
- $(Echo) "Building $(<F) code emitter with tblgen"
- $(Verb) $(LLVMTableGen) -gen-emitter -o $(call SYSPATH, $@) $<
-
$(TARGET:%=$(ObjDir)/%GenDAGISel.inc.tmp): \
$(ObjDir)/%GenDAGISel.inc.tmp : %.td $(ObjDir)/.dir $(LLVM_TBLGEN)
$(Echo) "Building $(<F) DAG instruction selector implementation with tblgen"
LEVEL := ../../..
LIBRARYNAME := llvm_executionengine
-UsedComponents := executionengine jit interpreter native
+UsedComponents := executionengine mcjit interpreter native
UsedOcamlInterfaces := llvm llvm_target
include ../Makefile.ocaml
/* Force the LLVM interpreter and JIT to be linked in. */
void llvm_initialize(void) {
LLVMLinkInInterpreter();
- LLVMLinkInJIT();
+ LLVMLinkInMCJIT();
}
/* unit -> bool */
**Output**: C++ code, implementing the target's CodeEmitter
class by overriding the virtual functions as ``<Target>CodeEmitter::function()``.
-**Usage**: Used to include directly at the end of ``<Target>CodeEmitter.cpp``, and
-with option `-mc-emitter` to be included in ``<Target>MCCodeEmitter.cpp``.
+**Usage**: Used to include directly at the end of ``<Target>MCCodeEmitter.cpp``.
RegisterInfo
------------
#include "BrainF.h"
#include "llvm/Bitcode/ReaderWriter.h"
+#include "llvm/ExecutionEngine/ExecutionEngine.h"
#include "llvm/ExecutionEngine/GenericValue.h"
-#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/CommandLine.h"
BitWriter
Core
ExecutionEngine
- JIT
MC
Support
nativecodegen
// Build engine with JIT
llvm::EngineBuilder factory(std::move(Owner));
factory.setEngineKind(llvm::EngineKind::JIT);
- factory.setAllocateGVsWithCode(false);
factory.setTargetOptions(Opts);
factory.setMCJITMemoryManager(MemMgr);
- factory.setUseMCJIT(true);
llvm::ExecutionEngine *executionEngine = factory.create();
{
Core
ExecutionEngine
Interpreter
- JIT
MC
Support
nativecodegen
#include "llvm/IR/Verifier.h"
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/Interpreter.h"
-#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Instructions.h"
Core
ExecutionEngine
Interpreter
- JIT
MC
Support
nativecodegen
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/Interpreter.h"
-#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
// Import result of execution:
outs() << "Result: " << gv.IntVal << "\n";
- EE->freeMachineCodeForFunction(FooF);
delete EE;
llvm_shutdown();
return 0;
Core
ExecutionEngine
InstCombine
- JIT
MC
ScalarOpts
Support
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
-#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
Core
ExecutionEngine
InstCombine
- JIT
MC
ScalarOpts
Support
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
-#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
Core
ExecutionEngine
InstCombine
- JIT
MC
ScalarOpts
Support
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
-#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
Core
ExecutionEngine
InstCombine
- JIT
MC
ScalarOpts
Support
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
-#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
-#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
std::string ErrStr;
ExecutionEngine *NewEngine = EngineBuilder(M)
.setErrorStr(&ErrStr)
- .setUseMCJIT(true)
.setMCJITMemoryManager(new HelpingMemoryManager(this))
.create();
if (!NewEngine) {
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
-#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/ExecutionEngine/MCJIT.h"
#include "llvm/ExecutionEngine/ObjectCache.h"
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
cl::desc("Dump IR from modules to stderr on shutdown"),
cl::init(false));
- cl::opt<bool> UseMCJIT(
- "use-mcjit", cl::desc("Use the MCJIT execution engine"),
- cl::init(true));
-
cl::opt<bool> EnableLazyCompilation(
"enable-lazy-compilation", cl::desc("Enable lazy compilation when using the MCJIT engine"),
cl::init(true));
virtual void dump();
};
-//===----------------------------------------------------------------------===//
-// Helper class for JIT execution engine
-//===----------------------------------------------------------------------===//
-
-class JITHelper : public BaseHelper {
-public:
- JITHelper(LLVMContext &Context) {
- // Make the module, which holds all the code.
- if (!InputIR.empty()) {
- TheModule = parseInputIR(InputIR, Context);
- } else {
- TheModule = new Module("my cool jit", Context);
- }
-
- // Create the JIT. This takes ownership of the module.
- std::string ErrStr;
- TheExecutionEngine = EngineBuilder(TheModule).setErrorStr(&ErrStr).create();
- if (!TheExecutionEngine) {
- fprintf(stderr, "Could not create ExecutionEngine: %s\n", ErrStr.c_str());
- exit(1);
- }
-
- TheFPM = new FunctionPassManager(TheModule);
-
- // Set up the optimizer pipeline. Start with registering info about how the
- // target lays out data structures.
- TheFPM->add(new DataLayout(*TheExecutionEngine->getDataLayout()));
- // Provide basic AliasAnalysis support for GVN.
- TheFPM->add(createBasicAliasAnalysisPass());
- // Promote allocas to registers.
- TheFPM->add(createPromoteMemoryToRegisterPass());
- // Do simple "peephole" optimizations and bit-twiddling optzns.
- TheFPM->add(createInstructionCombiningPass());
- // Reassociate expressions.
- TheFPM->add(createReassociatePass());
- // Eliminate Common SubExpressions.
- TheFPM->add(createGVNPass());
- // Simplify the control flow graph (deleting unreachable blocks, etc).
- TheFPM->add(createCFGSimplificationPass());
-
- TheFPM->doInitialization();
- }
-
- virtual ~JITHelper() {
- if (TheFPM)
- delete TheFPM;
- if (TheExecutionEngine)
- delete TheExecutionEngine;
- }
-
- virtual Function *getFunction(const std::string FnName) {
- assert(TheModule);
- return TheModule->getFunction(FnName);
- }
-
- virtual Module *getModuleForNewFunction() {
- assert(TheModule);
- return TheModule;
- }
-
- virtual void *getPointerToFunction(Function* F) {
- assert(TheExecutionEngine);
- return TheExecutionEngine->getPointerToFunction(F);
- }
-
- virtual void *getPointerToNamedFunction(const std::string &Name) {
- return TheExecutionEngine->getPointerToNamedFunction(Name);
- }
-
- virtual void runFPM(Function &F) {
- assert(TheFPM);
- TheFPM->run(F);
- }
-
- virtual void closeCurrentModule() {
- // This should never be called for JIT
- assert(false);
- }
-
- virtual void dump() {
- assert(TheModule);
- TheModule->dump();
- }
-
-private:
- Module *TheModule;
- ExecutionEngine *TheExecutionEngine;
- FunctionPassManager *TheFPM;
-};
-
//===----------------------------------------------------------------------===//
// MCJIT helper class
//===----------------------------------------------------------------------===//
std::string ErrStr;
ExecutionEngine *EE = EngineBuilder(M)
.setErrorStr(&ErrStr)
- .setUseMCJIT(true)
.setMCJITMemoryManager(new HelpingMemoryManager(this))
.create();
if (!EE) {
Value *OperandV = Operand->Codegen();
if (OperandV == 0) return 0;
Function *F;
- if (UseMCJIT)
- F = TheHelper->getFunction(MakeLegalFunctionName(std::string("unary")+Opcode));
- else
- F = TheHelper->getFunction(std::string("unary")+Opcode);
+ F = TheHelper->getFunction(
+ MakeLegalFunctionName(std::string("unary") + Opcode));
if (F == 0)
return ErrorV("Unknown unary operator");
// If it wasn't a builtin binary operator, it must be a user defined one. Emit
// a call to it.
Function *F;
- if (UseMCJIT)
- F = TheHelper->getFunction(MakeLegalFunctionName(std::string("binary")+Op));
- else
- F = TheHelper->getFunction(std::string("binary")+Op);
+ F = TheHelper->getFunction(MakeLegalFunctionName(std::string("binary")+Op));
assert(F && "binary operator not found!");
Value *Ops[] = { L, R };
Doubles, false);
std::string FnName;
- if (UseMCJIT)
- FnName = MakeLegalFunctionName(Name);
- else
- FnName = Name;
+ FnName = MakeLegalFunctionName(Name);
Module* M = TheHelper->getModuleForNewFunction();
Function *F = Function::Create(FT, Function::ExternalLinkage, FnName, M);
// Validate the generated code, checking for consistency.
verifyFunction(*TheFunction);
- // Optimize the function.
- if (!UseMCJIT)
- TheHelper->runFPM(*TheFunction);
-
return TheFunction;
}
static void HandleDefinition() {
if (FunctionAST *F = ParseDefinition()) {
- if (UseMCJIT && EnableLazyCompilation)
+ if (EnableLazyCompilation)
TheHelper->closeCurrentModule();
Function *LF = F->Codegen();
if (LF && VerboseOutput) {
int main(int argc, char **argv) {
InitializeNativeTarget();
- if (UseMCJIT) {
- InitializeNativeTargetAsmPrinter();
- InitializeNativeTargetAsmParser();
- }
+ InitializeNativeTargetAsmPrinter();
+ InitializeNativeTargetAsmParser();
LLVMContext &Context = getGlobalContext();
cl::ParseCommandLineOptions(argc, argv,
BinopPrecedence['*'] = 40; // highest.
// Make the Helper, which holds all the code.
- if (UseMCJIT)
- TheHelper = new MCJITHelper(Context);
- else
- TheHelper = new JITHelper(Context);
+ TheHelper = new MCJITHelper(Context);
// Prime the first token.
if (!SuppressPrompts)
std::string ErrStr;
ExecutionEngine *NewEngine = EngineBuilder(OpenModule)
.setErrorStr(&ErrStr)
- .setUseMCJIT(true)
.setMCJITMemoryManager(new HelpingMemoryManager(this))
.create();
if (!NewEngine) {
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
-#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
std::string ErrStr;
ExecutionEngine *NewEngine = EngineBuilder(M)
.setErrorStr(&ErrStr)
- .setUseMCJIT(true)
.setMCJITMemoryManager(new HelpingMemoryManager(this))
.create();
if (!NewEngine) {
Core
ExecutionEngine
Interpreter
- JIT
Support
nativecodegen
)
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/Interpreter.h"
-#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Instructions.h"
* @{
*/
-void LLVMLinkInJIT(void);
void LLVMLinkInMCJIT(void);
void LLVMLinkInInterpreter(void);
+++ /dev/null
-//===-- llvm/CodeGen/JITCodeEmitter.h - Code emission ----------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines an abstract interface that is used by the machine code
-// emission framework to output the code. This allows machine code emission to
-// be separated from concerns such as resolution of call targets, and where the
-// machine code will be written (memory or disk, f.e.).
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CODEGEN_JITCODEEMITTER_H
-#define LLVM_CODEGEN_JITCODEEMITTER_H
-
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/CodeGen/MachineCodeEmitter.h"
-#include "llvm/Support/DataTypes.h"
-#include "llvm/Support/MathExtras.h"
-#include <string>
-
-namespace llvm {
-
-class MachineBasicBlock;
-class MachineConstantPool;
-class MachineJumpTableInfo;
-class MachineFunction;
-class MachineModuleInfo;
-class MachineRelocation;
-class Value;
-class GlobalValue;
-class Function;
-
-/// JITCodeEmitter - This class defines two sorts of methods: those for
-/// emitting the actual bytes of machine code, and those for emitting auxiliary
-/// structures, such as jump tables, relocations, etc.
-///
-/// Emission of machine code is complicated by the fact that we don't (in
-/// general) know the size of the machine code that we're about to emit before
-/// we emit it. As such, we preallocate a certain amount of memory, and set the
-/// BufferBegin/BufferEnd pointers to the start and end of the buffer. As we
-/// emit machine instructions, we advance the CurBufferPtr to indicate the
-/// location of the next byte to emit. In the case of a buffer overflow (we
-/// need to emit more machine code than we have allocated space for), the
-/// CurBufferPtr will saturate to BufferEnd and ignore stores. Once the entire
-/// function has been emitted, the overflow condition is checked, and if it has
-/// occurred, more memory is allocated, and we reemit the code into it.
-///
-class JITCodeEmitter : public MachineCodeEmitter {
- void anchor() override;
-public:
- virtual ~JITCodeEmitter() {}
-
- /// startFunction - This callback is invoked when the specified function is
- /// about to be code generated. This initializes the BufferBegin/End/Ptr
- /// fields.
- ///
- void startFunction(MachineFunction &F) override = 0;
-
- /// finishFunction - This callback is invoked when the specified function has
- /// finished code generation. If a buffer overflow has occurred, this method
- /// returns true (the callee is required to try again), otherwise it returns
- /// false.
- ///
- bool finishFunction(MachineFunction &F) override = 0;
-
- /// allocIndirectGV - Allocates and fills storage for an indirect
- /// GlobalValue, and returns the address.
- virtual void *allocIndirectGV(const GlobalValue *GV,
- const uint8_t *Buffer, size_t Size,
- unsigned Alignment) = 0;
-
- /// emitByte - This callback is invoked when a byte needs to be written to the
- /// output stream.
- ///
- void emitByte(uint8_t B) {
- if (CurBufferPtr != BufferEnd)
- *CurBufferPtr++ = B;
- }
-
- /// emitWordLE - This callback is invoked when a 32-bit word needs to be
- /// written to the output stream in little-endian format.
- ///
- void emitWordLE(uint32_t W) {
- if (4 <= BufferEnd-CurBufferPtr) {
- *CurBufferPtr++ = (uint8_t)(W >> 0);
- *CurBufferPtr++ = (uint8_t)(W >> 8);
- *CurBufferPtr++ = (uint8_t)(W >> 16);
- *CurBufferPtr++ = (uint8_t)(W >> 24);
- } else {
- CurBufferPtr = BufferEnd;
- }
- }
-
- /// emitWordBE - This callback is invoked when a 32-bit word needs to be
- /// written to the output stream in big-endian format.
- ///
- void emitWordBE(uint32_t W) {
- if (4 <= BufferEnd-CurBufferPtr) {
- *CurBufferPtr++ = (uint8_t)(W >> 24);
- *CurBufferPtr++ = (uint8_t)(W >> 16);
- *CurBufferPtr++ = (uint8_t)(W >> 8);
- *CurBufferPtr++ = (uint8_t)(W >> 0);
- } else {
- CurBufferPtr = BufferEnd;
- }
- }
-
- /// emitDWordLE - This callback is invoked when a 64-bit word needs to be
- /// written to the output stream in little-endian format.
- ///
- void emitDWordLE(uint64_t W) {
- if (8 <= BufferEnd-CurBufferPtr) {
- *CurBufferPtr++ = (uint8_t)(W >> 0);
- *CurBufferPtr++ = (uint8_t)(W >> 8);
- *CurBufferPtr++ = (uint8_t)(W >> 16);
- *CurBufferPtr++ = (uint8_t)(W >> 24);
- *CurBufferPtr++ = (uint8_t)(W >> 32);
- *CurBufferPtr++ = (uint8_t)(W >> 40);
- *CurBufferPtr++ = (uint8_t)(W >> 48);
- *CurBufferPtr++ = (uint8_t)(W >> 56);
- } else {
- CurBufferPtr = BufferEnd;
- }
- }
-
- /// emitDWordBE - This callback is invoked when a 64-bit word needs to be
- /// written to the output stream in big-endian format.
- ///
- void emitDWordBE(uint64_t W) {
- if (8 <= BufferEnd-CurBufferPtr) {
- *CurBufferPtr++ = (uint8_t)(W >> 56);
- *CurBufferPtr++ = (uint8_t)(W >> 48);
- *CurBufferPtr++ = (uint8_t)(W >> 40);
- *CurBufferPtr++ = (uint8_t)(W >> 32);
- *CurBufferPtr++ = (uint8_t)(W >> 24);
- *CurBufferPtr++ = (uint8_t)(W >> 16);
- *CurBufferPtr++ = (uint8_t)(W >> 8);
- *CurBufferPtr++ = (uint8_t)(W >> 0);
- } else {
- CurBufferPtr = BufferEnd;
- }
- }
-
- /// emitAlignment - Move the CurBufferPtr pointer up to the specified
- /// alignment (saturated to BufferEnd of course).
- void emitAlignment(unsigned Alignment) {
- if (Alignment == 0) Alignment = 1;
- uint8_t *NewPtr = (uint8_t*)RoundUpToAlignment((uintptr_t)CurBufferPtr,
- Alignment);
- CurBufferPtr = std::min(NewPtr, BufferEnd);
- }
-
- /// emitAlignmentWithFill - Similar to emitAlignment, except that the
- /// extra bytes are filled with the provided byte.
- void emitAlignmentWithFill(unsigned Alignment, uint8_t Fill) {
- if (Alignment == 0) Alignment = 1;
- uint8_t *NewPtr = (uint8_t*)RoundUpToAlignment((uintptr_t)CurBufferPtr,
- Alignment);
- // Fail if we don't have room.
- if (NewPtr > BufferEnd) {
- CurBufferPtr = BufferEnd;
- return;
- }
- while (CurBufferPtr < NewPtr) {
- *CurBufferPtr++ = Fill;
- }
- }
-
- /// emitULEB128Bytes - This callback is invoked when a ULEB128 needs to be
- /// written to the output stream.
- void emitULEB128Bytes(uint64_t Value, unsigned PadTo = 0) {
- do {
- uint8_t Byte = Value & 0x7f;
- Value >>= 7;
- if (Value || PadTo != 0) Byte |= 0x80;
- emitByte(Byte);
- } while (Value);
-
- if (PadTo) {
- do {
- uint8_t Byte = (PadTo > 1) ? 0x80 : 0x0;
- emitByte(Byte);
- } while (--PadTo);
- }
- }
-
- /// emitSLEB128Bytes - This callback is invoked when a SLEB128 needs to be
- /// written to the output stream.
- void emitSLEB128Bytes(int64_t Value) {
- int32_t Sign = Value >> (8 * sizeof(Value) - 1);
- bool IsMore;
-
- do {
- uint8_t Byte = Value & 0x7f;
- Value >>= 7;
- IsMore = Value != Sign || ((Byte ^ Sign) & 0x40) != 0;
- if (IsMore) Byte |= 0x80;
- emitByte(Byte);
- } while (IsMore);
- }
-
- /// emitString - This callback is invoked when a String needs to be
- /// written to the output stream.
- void emitString(const std::string &String) {
- for (size_t i = 0, N = String.size(); i < N; ++i) {
- uint8_t C = String[i];
- emitByte(C);
- }
- emitByte(0);
- }
-
- /// emitInt32 - Emit a int32 directive.
- void emitInt32(uint32_t Value) {
- if (4 <= BufferEnd-CurBufferPtr) {
- *((uint32_t*)CurBufferPtr) = Value;
- CurBufferPtr += 4;
- } else {
- CurBufferPtr = BufferEnd;
- }
- }
-
- /// emitInt64 - Emit a int64 directive.
- void emitInt64(uint64_t Value) {
- if (8 <= BufferEnd-CurBufferPtr) {
- *((uint64_t*)CurBufferPtr) = Value;
- CurBufferPtr += 8;
- } else {
- CurBufferPtr = BufferEnd;
- }
- }
-
- /// emitInt32At - Emit the Int32 Value in Addr.
- void emitInt32At(uintptr_t *Addr, uintptr_t Value) {
- if (Addr >= (uintptr_t*)BufferBegin && Addr < (uintptr_t*)BufferEnd)
- (*(uint32_t*)Addr) = (uint32_t)Value;
- }
-
- /// emitInt64At - Emit the Int64 Value in Addr.
- void emitInt64At(uintptr_t *Addr, uintptr_t Value) {
- if (Addr >= (uintptr_t*)BufferBegin && Addr < (uintptr_t*)BufferEnd)
- (*(uint64_t*)Addr) = (uint64_t)Value;
- }
-
-
- /// emitLabel - Emits a label
- void emitLabel(MCSymbol *Label) override = 0;
-
- /// allocateSpace - Allocate a block of space in the current output buffer,
- /// returning null (and setting conditions to indicate buffer overflow) on
- /// failure. Alignment is the alignment in bytes of the buffer desired.
- void *allocateSpace(uintptr_t Size, unsigned Alignment) override {
- emitAlignment(Alignment);
- void *Result;
-
- // Check for buffer overflow.
- if (Size >= (uintptr_t)(BufferEnd-CurBufferPtr)) {
- CurBufferPtr = BufferEnd;
- Result = nullptr;
- } else {
- // Allocate the space.
- Result = CurBufferPtr;
- CurBufferPtr += Size;
- }
-
- return Result;
- }
-
- /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace,
- /// this method does not allocate memory in the current output buffer,
- /// because a global may live longer than the current function.
- virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment) = 0;
-
- /// StartMachineBasicBlock - This should be called by the target when a new
- /// basic block is about to be emitted. This way the MCE knows where the
- /// start of the block is, and can implement getMachineBasicBlockAddress.
- void StartMachineBasicBlock(MachineBasicBlock *MBB) override = 0;
-
- /// getCurrentPCValue - This returns the address that the next emitted byte
- /// will be output to.
- ///
- uintptr_t getCurrentPCValue() const override {
- return (uintptr_t)CurBufferPtr;
- }
-
- /// getCurrentPCOffset - Return the offset from the start of the emitted
- /// buffer that we are currently writing to.
- uintptr_t getCurrentPCOffset() const override {
- return CurBufferPtr-BufferBegin;
- }
-
- /// earlyResolveAddresses - True if the code emitter can use symbol addresses
- /// during code emission time. The JIT is capable of doing this because it
- /// creates jump tables or constant pools in memory on the fly while the
- /// object code emitters rely on a linker to have real addresses and should
- /// use relocations instead.
- bool earlyResolveAddresses() const override { return true; }
-
- /// addRelocation - Whenever a relocatable address is needed, it should be
- /// noted with this interface.
- void addRelocation(const MachineRelocation &MR) override = 0;
-
- /// FIXME: These should all be handled with relocations!
-
- /// getConstantPoolEntryAddress - Return the address of the 'Index' entry in
- /// the constant pool that was last emitted with the emitConstantPool method.
- ///
- uintptr_t getConstantPoolEntryAddress(unsigned Index) const override = 0;
-
- /// getJumpTableEntryAddress - Return the address of the jump table with index
- /// 'Index' in the function that last called initJumpTableInfo.
- ///
- uintptr_t getJumpTableEntryAddress(unsigned Index) const override = 0;
-
- /// getMachineBasicBlockAddress - Return the address of the specified
- /// MachineBasicBlock, only usable after the label for the MBB has been
- /// emitted.
- ///
- uintptr_t
- getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override = 0;
-
- /// getLabelAddress - Return the address of the specified Label, only usable
- /// after the Label has been emitted.
- ///
- uintptr_t getLabelAddress(MCSymbol *Label) const override = 0;
-
- /// Specifies the MachineModuleInfo object. This is used for exception handling
- /// purposes.
- void setModuleInfo(MachineModuleInfo* Info) override = 0;
-
- /// getLabelLocations - Return the label locations map of the label IDs to
- /// their address.
- virtual DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() {
- return nullptr;
- }
-};
-
-} // End llvm namespace
-
-#endif
/// getMemoryforGV - Allocate memory for a global variable.
virtual char *getMemoryForGV(const GlobalVariable *GV);
- // To avoid having libexecutionengine depend on the JIT and interpreter
- // libraries, the execution engine implementations set these functions to ctor
- // pointers at startup time if they are linked in.
- static ExecutionEngine *(*JITCtor)(
- std::unique_ptr<Module> M,
- std::string *ErrorStr,
- JITMemoryManager *JMM,
- bool GVsWithCode,
- TargetMachine *TM);
static ExecutionEngine *(*MCJITCtor)(
std::unique_ptr<Module> M,
std::string *ErrorStr,
/// getFunctionAddress instead.
virtual void *getPointerToFunction(Function *F) = 0;
- /// getPointerToBasicBlock - The different EE's represent basic blocks in
- /// different ways. Return the representation for a blockaddress of the
- /// specified block.
- ///
- /// This function will not be implemented for the MCJIT execution engine.
- virtual void *getPointerToBasicBlock(BasicBlock *BB) = 0;
-
/// getPointerToFunctionOrStub - If the specified function has been
/// code-gen'd, return a pointer to the function. If not, compile it, or use
/// a stub to implement lazy compilation if available. See
void InitializeMemory(const Constant *Init, void *Addr);
- /// recompileAndRelinkFunction - This method is used to force a function which
- /// has already been compiled to be compiled again, possibly after it has been
- /// modified. Then the entry to the old copy is overwritten with a branch to
- /// the new copy. If there was no old copy, this acts just like
- /// VM::getPointerToFunction().
- virtual void *recompileAndRelinkFunction(Function *F) = 0;
-
- /// freeMachineCodeForFunction - Release memory in the ExecutionEngine
- /// corresponding to the machine code emitted to execute this function, useful
- /// for garbage-collecting generated code.
- virtual void freeMachineCodeForFunction(Function *F) = 0;
-
/// getOrEmitGlobalVariable - Return the address of the specified global
/// variable, possibly emitting it to memory if needed. This is used by the
/// Emitter.
CodeGenOpt::Level OptLevel;
RTDyldMemoryManager *MCJMM;
JITMemoryManager *JMM;
- bool AllocateGVsWithCode;
TargetOptions Options;
Reloc::Model RelocModel;
CodeModel::Model CMModel;
std::string MArch;
std::string MCPU;
SmallVector<std::string, 4> MAttrs;
- bool UseMCJIT;
bool VerifyModules;
/// InitEngine - Does the common initialization of default options.
return *this;
}
- /// setAllocateGVsWithCode - Sets whether global values should be allocated
- /// into the same buffer as code. For most applications this should be set
- /// to false. Allocating globals with code breaks freeMachineCodeForFunction
- /// and is probably unsafe and bad for performance. However, we have clients
- /// who depend on this behavior, so we must support it. This option defaults
- /// to false so that users of the new API can safely use the new memory
- /// manager and free machine code.
- EngineBuilder &setAllocateGVsWithCode(bool a) {
- AllocateGVsWithCode = a;
- return *this;
- }
-
/// setMArch - Override the architecture set by the Module's triple.
EngineBuilder &setMArch(StringRef march) {
MArch.assign(march.begin(), march.end());
return *this;
}
- /// setUseMCJIT - Set whether the MC-JIT implementation should be used
- /// (experimental).
- EngineBuilder &setUseMCJIT(bool Value) {
- UseMCJIT = Value;
- return *this;
- }
-
/// setVerifyModules - Set whether the JIT implementation should verify
/// IR modules during compilation.
EngineBuilder &setVerifyModules(bool Verify) {
+++ /dev/null
-//===-- JIT.h - Abstract Execution Engine Interface -------------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file forces the JIT to link in on certain operating systems.
-// (Windows).
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_EXECUTIONENGINE_JIT_H
-#define LLVM_EXECUTIONENGINE_JIT_H
-
-#include "llvm/ExecutionEngine/ExecutionEngine.h"
-#include <cstdlib>
-
-extern "C" void LLVMLinkInJIT();
-
-namespace {
- struct ForceJITLinking {
- ForceJITLinking() {
- // We must reference JIT in such a way that compilers will not
- // delete it all as dead code, even with whole program optimization,
- // yet is effectively a NO-OP. As the compiler isn't smart enough
- // to know that getenv() never returns -1, this will do the job.
- if (std::getenv("bar") != (char*) -1)
- return;
-
- LLVMLinkInJIT();
- }
- } ForceJITLinking;
-}
-
-#endif
+++ /dev/null
-//===- Target/TargetJITInfo.h - Target Information for JIT ------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file exposes an abstract interface used by the Just-In-Time code
-// generator to perform target-specific activities, such as emitting stubs. If
-// a TargetMachine supports JIT code generation, it should provide one of these
-// objects through the getJITInfo() method.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TARGET_TARGETJITINFO_H
-#define LLVM_TARGET_TARGETJITINFO_H
-
-#include "llvm/Support/DataTypes.h"
-#include "llvm/Support/ErrorHandling.h"
-#include <cassert>
-
-namespace llvm {
- class Function;
- class GlobalValue;
- class JITCodeEmitter;
- class MachineRelocation;
-
- /// TargetJITInfo - Target specific information required by the Just-In-Time
- /// code generator.
- class TargetJITInfo {
- virtual void anchor();
- public:
- virtual ~TargetJITInfo() {}
-
- /// replaceMachineCodeForFunction - Make it so that calling the function
- /// whose machine code is at OLD turns into a call to NEW, perhaps by
- /// overwriting OLD with a branch to NEW. This is used for self-modifying
- /// code.
- ///
- virtual void replaceMachineCodeForFunction(void *Old, void *New) = 0;
-
- /// emitGlobalValueIndirectSym - Use the specified JITCodeEmitter object
- /// to emit an indirect symbol which contains the address of the specified
- /// ptr.
- virtual void *emitGlobalValueIndirectSym(const GlobalValue* GV, void *ptr,
- JITCodeEmitter &JCE) {
- llvm_unreachable("This target doesn't implement "
- "emitGlobalValueIndirectSym!");
- }
-
- /// Records the required size and alignment for a call stub in bytes.
- struct StubLayout {
- size_t Size;
- size_t Alignment;
- };
- /// Returns the maximum size and alignment for a call stub on this target.
- virtual StubLayout getStubLayout() {
- llvm_unreachable("This target doesn't implement getStubLayout!");
- }
-
- /// emitFunctionStub - Use the specified JITCodeEmitter object to emit a
- /// small native function that simply calls the function at the specified
- /// address. The JITCodeEmitter must already have storage allocated for the
- /// stub. Return the address of the resultant function, which may have been
- /// aligned from the address the JCE was set up to emit at.
- virtual void *emitFunctionStub(const Function* F, void *Target,
- JITCodeEmitter &JCE) {
- llvm_unreachable("This target doesn't implement emitFunctionStub!");
- }
-
- /// getPICJumpTableEntry - Returns the value of the jumptable entry for the
- /// specific basic block.
- virtual uintptr_t getPICJumpTableEntry(uintptr_t BB, uintptr_t JTBase) {
- llvm_unreachable("This target doesn't implement getPICJumpTableEntry!");
- }
-
- /// LazyResolverFn - This typedef is used to represent the function that
- /// unresolved call points should invoke. This is a target specific
- /// function that knows how to walk the stack and find out which stub the
- /// call is coming from.
- typedef void (*LazyResolverFn)();
-
- /// JITCompilerFn - This typedef is used to represent the JIT function that
- /// lazily compiles the function corresponding to a stub. The JIT keeps
- /// track of the mapping between stubs and LLVM Functions, the target
- /// provides the ability to figure out the address of a stub that is called
- /// by the LazyResolverFn.
- typedef void* (*JITCompilerFn)(void *);
-
- /// getLazyResolverFunction - This method is used to initialize the JIT,
- /// giving the target the function that should be used to compile a
- /// function, and giving the JIT the target function used to do the lazy
- /// resolving.
- virtual LazyResolverFn getLazyResolverFunction(JITCompilerFn) {
- llvm_unreachable("Not implemented for this target!");
- }
-
- /// relocate - Before the JIT can run a block of code that has been emitted,
- /// it must rewrite the code to contain the actual addresses of any
- /// referenced global symbols.
- virtual void relocate(void *Function, MachineRelocation *MR,
- unsigned NumRelocs, unsigned char* GOTBase) {
- assert(NumRelocs == 0 && "This target does not have relocations!");
- }
-
- /// allocateThreadLocalMemory - Each target has its own way of
- /// handling thread local variables. This method returns a value only
- /// meaningful to the target.
- virtual char* allocateThreadLocalMemory(size_t size) {
- llvm_unreachable("This target does not implement thread local storage!");
- }
-
- /// needsGOT - Allows a target to specify that it would like the
- /// JIT to manage a GOT for it.
- bool needsGOT() const { return useGOT; }
-
- /// hasCustomConstantPool - Allows a target to specify that constant
- /// pool address resolution is handled by the target.
- virtual bool hasCustomConstantPool() const { return false; }
-
- /// hasCustomJumpTables - Allows a target to specify that jumptables
- /// are emitted by the target.
- virtual bool hasCustomJumpTables() const { return false; }
-
- /// allocateSeparateGVMemory - If true, globals should be placed in
- /// separately allocated heap memory rather than in the same
- /// code memory allocated by JITCodeEmitter.
- virtual bool allocateSeparateGVMemory() const { return false; }
- protected:
- bool useGOT;
- };
-} // End llvm namespace
-
-#endif
return UseUnderscoreLongJmp;
}
- /// Return whether the target can generate code for jump tables.
- bool supportJumpTables() const {
- return SupportJumpTables;
- }
-
/// Return integer threshold on number of blocks to use jump tables rather
/// than if sequence.
int getMinimumJumpTableEntries() const {
UseUnderscoreLongJmp = Val;
}
- /// Indicate whether the target can generate code for jump tables.
- void setSupportJumpTables(bool Val) {
- SupportJumpTables = Val;
- }
-
/// Indicate the number of blocks to generate jump tables rather than if
/// sequence.
void setMinimumJumpTableEntries(int Val) {
/// Defaults to false.
bool UseUnderscoreLongJmp;
- /// Whether the target can generate code for jumptables. If it's not true,
- /// then each jumptable must be lowered into if-then-else's.
- bool SupportJumpTables;
-
/// Number of blocks threshold to use jump tables.
int MinimumJumpTableEntries;
namespace llvm {
class InstrItineraryData;
-class JITCodeEmitter;
class GlobalValue;
class Mangler;
class MCAsmInfo;
class TargetLibraryInfo;
class TargetFrameLowering;
class TargetIntrinsicInfo;
-class TargetJITInfo;
class TargetLowering;
class TargetPassConfig;
class TargetRegisterInfo;
virtual const TargetSubtargetInfo *getSubtargetImpl() const {
return nullptr;
}
- TargetSubtargetInfo *getSubtargetImpl() {
- const TargetMachine *TM = this;
- return const_cast<TargetSubtargetInfo *>(TM->getSubtargetImpl());
- }
/// getSubtarget - This method returns a pointer to the specified type of
/// TargetSubtargetInfo. In debug builds, it verifies that the object being
return true;
}
- /// addPassesToEmitMachineCode - Add passes to the specified pass manager to
- /// get machine code emitted. This uses a JITCodeEmitter object to handle
- /// actually outputting the machine code and resolving things like the address
- /// of functions. This method returns true if machine code emission is
- /// not supported.
- ///
- virtual bool addPassesToEmitMachineCode(PassManagerBase &,
- JITCodeEmitter &,
- bool /*DisableVerify*/ = true) {
- return true;
- }
-
/// addPassesToEmitMC - Add passes to the specified pass manager to get
/// machine code emitted with the MCJIT. This method returns true if machine
/// code is not supported. It fills the MCContext Ctx pointer which can be
AnalysisID StartAfter = nullptr,
AnalysisID StopAfter = nullptr) override;
- /// addPassesToEmitMachineCode - Add passes to the specified pass manager to
- /// get machine code emitted. This uses a JITCodeEmitter object to handle
- /// actually outputting the machine code and resolving things like the address
- /// of functions. This method returns true if machine code emission is
- /// not supported.
- ///
- bool addPassesToEmitMachineCode(PassManagerBase &PM, JITCodeEmitter &MCE,
- bool DisableVerify = true) override;
-
/// addPassesToEmitMC - Add passes to the specified pass manager to get
/// machine code emitted with the MCJIT. This method returns true if machine
/// code is not supported. It fills the MCContext Ctx pointer which can be
///
bool addPassesToEmitMC(PassManagerBase &PM, MCContext *&Ctx,
raw_ostream &OS, bool DisableVerify = true) override;
-
- /// addCodeEmitter - This pass should be overridden by the target to add a
- /// code emitter, if supported. If this is not supported, 'true' should be
- /// returned.
- virtual bool addCodeEmitter(PassManagerBase &,
- JITCodeEmitter &) {
- return true;
- }
};
} // End llvm namespace
class SUnit;
class TargetFrameLowering;
class TargetInstrInfo;
-class TargetJITInfo;
class TargetLowering;
class TargetRegisterClass;
class TargetRegisterInfo;
///
virtual const TargetRegisterInfo *getRegisterInfo() const { return nullptr; }
- /// getJITInfo - If this target supports a JIT, return information for it,
- /// otherwise return null.
- ///
- virtual TargetJITInfo *getJITInfo() { return nullptr; }
-
/// getInstrItineraryData - Returns instruction itinerary data for the target
/// or specific subtarget.
///
bool BasicTTI::shouldBuildLookupTables() const {
const TargetLoweringBase *TLI = getTLI();
- return TLI->supportJumpTables() &&
- (TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
- TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other));
+ return TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
+ TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
}
bool BasicTTI::haveFastSqrt(Type *Ty) const {
InlineSpiller.cpp
InterferenceCache.cpp
IntrinsicLowering.cpp
- JITCodeEmitter.cpp
JumpInstrTables.cpp
LLVMTargetMachine.cpp
LatencyPriorityQueue.cpp
+++ /dev/null
-//===-- llvm/CodeGen/JITCodeEmitter.cpp - Code emission --------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/CodeGen/JITCodeEmitter.h"
-
-using namespace llvm;
-
-void JITCodeEmitter::anchor() { }
return false;
}
-/// addPassesToEmitMachineCode - Add passes to the specified pass manager to
-/// get machine code emitted. This uses a JITCodeEmitter object to handle
-/// actually outputting the machine code and resolving things like the address
-/// of functions. This method should return true if machine code emission is
-/// not supported.
-///
-bool LLVMTargetMachine::addPassesToEmitMachineCode(PassManagerBase &PM,
- JITCodeEmitter &JCE,
- bool DisableVerify) {
- // Add common CodeGen passes.
- MCContext *Context = addPassesToGenerateCode(this, PM, DisableVerify, nullptr,
- nullptr);
- if (!Context)
- return true;
-
- addCodeEmitter(PM, JCE);
-
- return false; // success!
-}
-
/// addPassesToEmitMC - Add passes to the specified pass manager to get
/// machine code emitted with the MCJIT. This method returns true if machine
/// code is not supported. It fills the MCContext Ctx pointer which can be
}
static inline bool areJTsAllowed(const TargetLowering &TLI) {
- return TLI.supportJumpTables() &&
- (TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
- TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other));
+ return TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
+ TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
}
static APInt ComputeRange(const APInt &First, const APInt &Last) {
PrefLoopAlignment = 0;
MinStackArgumentAlignment = 1;
InsertFencesForAtomic = false;
- SupportJumpTables = true;
MinimumJumpTableEntries = 4;
InitLibcallNames(LibcallRoutineNames, Triple(TM.getTargetTriple()));
)
add_subdirectory(Interpreter)
-add_subdirectory(JIT)
add_subdirectory(MCJIT)
add_subdirectory(RuntimeDyld)
void ObjectBuffer::anchor() {}
void ObjectBufferStream::anchor() {}
-ExecutionEngine *(*ExecutionEngine::JITCtor)(
- std::unique_ptr<Module> M,
- std::string *ErrorStr,
- JITMemoryManager *JMM,
- bool GVsWithCode,
- TargetMachine *TM) = nullptr;
ExecutionEngine *(*ExecutionEngine::MCJITCtor)(
std::unique_ptr<Module >M,
std::string *ErrorStr,
MCJMM = nullptr;
JMM = nullptr;
Options = TargetOptions();
- AllocateGVsWithCode = false;
RelocModel = Reloc::Default;
CMModel = CodeModel::JITDefault;
- UseMCJIT = false;
// IR module verification is enabled by default in debug builds, and disabled
// by default in release builds.
return nullptr;
}
}
-
- if (MCJMM && ! UseMCJIT) {
- if (ErrorStr)
- *ErrorStr =
- "Cannot create a legacy JIT with a runtime dyld memory "
- "manager.";
- return nullptr;
- }
// Unless the interpreter was explicitly selected or the JIT is not linked,
// try making a JIT.
}
ExecutionEngine *EE = nullptr;
- if (UseMCJIT && ExecutionEngine::MCJITCtor)
+ if (ExecutionEngine::MCJITCtor)
EE = ExecutionEngine::MCJITCtor(std::move(M), ErrorStr,
MCJMM ? MCJMM : JMM, TheTM.release());
- else if (ExecutionEngine::JITCtor)
- EE = ExecutionEngine::JITCtor(std::move(M), ErrorStr, JMM,
- AllocateGVsWithCode, TheTM.release());
-
if (EE) {
EE->setVerifyModules(VerifyModules);
return EE;
return nullptr;
}
- if ((WhichEngine & EngineKind::JIT) && !ExecutionEngine::JITCtor &&
- !ExecutionEngine::MCJITCtor) {
+ if ((WhichEngine & EngineKind::JIT) && !ExecutionEngine::MCJITCtor) {
if (ErrorStr)
*ErrorStr = "JIT has not been linked in.";
}
Result = PTOGV(getPointerToFunctionOrStub(const_cast<Function*>(F)));
else if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(C))
Result = PTOGV(getOrEmitGlobalVariable(const_cast<GlobalVariable*>(GV)));
- else if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
- Result = PTOGV(getPointerToBasicBlock(const_cast<BasicBlock*>(
- BA->getBasicBlock())));
else
llvm_unreachable("Unknown constant pointer type!");
break;
EngineBuilder builder(std::unique_ptr<Module>(unwrap(M)));
builder.setEngineKind(EngineKind::JIT)
.setErrorStr(&Error)
- .setUseMCJIT(true)
.setOptLevel((CodeGenOpt::Level)options.OptLevel)
.setCodeModel(unwrap(options.CodeModel))
.setTargetOptions(targetOptions);
}
void LLVMFreeMachineCodeForFunction(LLVMExecutionEngineRef EE, LLVMValueRef F) {
- unwrap(EE)->freeMachineCodeForFunction(unwrap<Function>(F));
}
void LLVMAddModule(LLVMExecutionEngineRef EE, LLVMModuleRef M){
void *LLVMRecompileAndRelinkFunction(LLVMExecutionEngineRef EE,
LLVMValueRef Fn) {
- return unwrap(EE)->recompileAndRelinkFunction(unwrap<Function>(Fn));
+ return nullptr;
}
LLVMTargetDataRef LLVMGetExecutionEngineTargetData(LLVMExecutionEngineRef EE) {
return nullptr;
}
- /// recompileAndRelinkFunction - For the interpreter, functions are always
- /// up-to-date.
- ///
- void *recompileAndRelinkFunction(Function *F) override {
- return getPointerToFunction(F);
- }
-
- /// freeMachineCodeForFunction - The interpreter does not generate any code.
- ///
- void freeMachineCodeForFunction(Function *F) override { }
-
// Methods used to execute code:
// Place a call on the stack
void callFunction(Function *F, const std::vector<GenericValue> &ArgVals);
void SwitchToNewBasicBlock(BasicBlock *Dest, ExecutionContext &SF);
void *getPointerToFunction(Function *F) override { return (void*)F; }
- void *getPointerToBasicBlock(BasicBlock *BB) override { return (void*)BB; }
void initializeExecutionEngine() { }
void initializeExternalFunctions();
+++ /dev/null
-# TODO: Support other architectures. See Makefile.
-add_definitions(-DENABLE_X86_JIT)
-
-add_llvm_library(LLVMJIT
- JIT.cpp
- JITEmitter.cpp
- JITMemoryManager.cpp
- )
+++ /dev/null
-//===-- JIT.cpp - LLVM Just in Time Compiler ------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This tool implements a just-in-time compiler for LLVM, allowing direct
-// execution of LLVM bitcode in an efficient manner.
-//
-//===----------------------------------------------------------------------===//
-
-#include "JIT.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/CodeGen/JITCodeEmitter.h"
-#include "llvm/CodeGen/MachineCodeInfo.h"
-#include "llvm/Config/config.h"
-#include "llvm/ExecutionEngine/GenericValue.h"
-#include "llvm/ExecutionEngine/JITEventListener.h"
-#include "llvm/ExecutionEngine/JITMemoryManager.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/GlobalVariable.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Support/Dwarf.h"
-#include "llvm/Support/DynamicLibrary.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/ManagedStatic.h"
-#include "llvm/Support/MutexGuard.h"
-#include "llvm/Target/TargetJITInfo.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetSubtargetInfo.h"
-
-using namespace llvm;
-
-#ifdef __APPLE__
-// Apple gcc defaults to -fuse-cxa-atexit (i.e. calls __cxa_atexit instead
-// of atexit). It passes the address of linker generated symbol __dso_handle
-// to the function.
-// This configuration change happened at version 5330.
-# include <AvailabilityMacros.h>
-# if defined(MAC_OS_X_VERSION_10_4) && \
- ((MAC_OS_X_VERSION_MIN_REQUIRED > MAC_OS_X_VERSION_10_4) || \
- (MAC_OS_X_VERSION_MIN_REQUIRED == MAC_OS_X_VERSION_10_4 && \
- __APPLE_CC__ >= 5330))
-# ifndef HAVE___DSO_HANDLE
-# define HAVE___DSO_HANDLE 1
-# endif
-# endif
-#endif
-
-#if HAVE___DSO_HANDLE
-extern void *__dso_handle __attribute__ ((__visibility__ ("hidden")));
-#endif
-
-namespace {
-
-static struct RegisterJIT {
- RegisterJIT() { JIT::Register(); }
-} JITRegistrator;
-
-}
-
-extern "C" void LLVMLinkInJIT() {
-}
-
-/// This is the factory method for creating a JIT for the current machine, it
-/// does not fall back to the interpreter.
-ExecutionEngine *JIT::createJIT(std::unique_ptr<Module> M,
- std::string *ErrorStr,
- JITMemoryManager *JMM,
- bool GVsWithCode,
- TargetMachine *TM) {
- // Try to register the program as a source of symbols to resolve against.
- //
- // FIXME: Don't do this here.
- sys::DynamicLibrary::LoadLibraryPermanently(nullptr, nullptr);
-
- // If the target supports JIT code generation, create the JIT.
- if (TargetJITInfo *TJ = TM->getSubtargetImpl()->getJITInfo()) {
- return new JIT(std::move(M), *TM, *TJ, JMM, GVsWithCode);
- } else {
- if (ErrorStr)
- *ErrorStr = "target does not support JIT code generation";
- return nullptr;
- }
-}
-
-namespace {
-/// This class supports the global getPointerToNamedFunction(), which allows
-/// bugpoint or gdb users to search for a function by name without any context.
-class JitPool {
- SmallPtrSet<JIT*, 1> JITs; // Optimize for process containing just 1 JIT.
- mutable sys::Mutex Lock;
-public:
- void Add(JIT *jit) {
- MutexGuard guard(Lock);
- JITs.insert(jit);
- }
- void Remove(JIT *jit) {
- MutexGuard guard(Lock);
- JITs.erase(jit);
- }
- void *getPointerToNamedFunction(const char *Name) const {
- MutexGuard guard(Lock);
- assert(JITs.size() != 0 && "No Jit registered");
- //search function in every instance of JIT
- for (JIT *Jit : JITs) {
- if (Function *F = Jit->FindFunctionNamed(Name))
- return Jit->getPointerToFunction(F);
- }
- // The function is not available : fallback on the first created (will
- // search in symbol of the current program/library)
- return (*JITs.begin())->getPointerToNamedFunction(Name);
- }
-};
-ManagedStatic<JitPool> AllJits;
-}
-extern "C" {
- // getPointerToNamedFunction - This function is used as a global wrapper to
- // JIT::getPointerToNamedFunction for the purpose of resolving symbols when
- // bugpoint is debugging the JIT. In that scenario, we are loading an .so and
- // need to resolve function(s) that are being mis-codegenerated, so we need to
- // resolve their addresses at runtime, and this is the way to do it.
- void *getPointerToNamedFunction(const char *Name) {
- return AllJits->getPointerToNamedFunction(Name);
- }
-}
-
-JIT::JIT(std::unique_ptr<Module> M, TargetMachine &tm, TargetJITInfo &tji,
- JITMemoryManager *jmm, bool GVsWithCode)
- : ExecutionEngine(std::move(M)), TM(tm), TJI(tji),
- JMM(jmm ? jmm : JITMemoryManager::CreateDefaultMemManager()),
- AllocateGVsWithCode(GVsWithCode), isAlreadyCodeGenerating(false) {
- setDataLayout(TM.getSubtargetImpl()->getDataLayout());
-
- Module *Mod = Modules.back().get();
- jitstate = new JITState(Mod);
-
- // Initialize JCE
- JCE = createEmitter(*this, JMM, TM);
-
- // Register in global list of all JITs.
- AllJits->Add(this);
-
- // Add target data
- MutexGuard locked(lock);
- FunctionPassManager &PM = jitstate->getPM();
- Mod->setDataLayout(TM.getSubtargetImpl()->getDataLayout());
- PM.add(new DataLayoutPass(Mod));
-
- // Turn the machine code intermediate representation into bytes in memory that
- // may be executed.
- if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) {
- report_fatal_error("Target does not support machine code emission!");
- }
-
- // Initialize passes.
- PM.doInitialization();
-}
-
-JIT::~JIT() {
- // Cleanup.
- AllJits->Remove(this);
- delete jitstate;
- delete JCE;
- // JMM is a ownership of JCE, so we no need delete JMM here.
- delete &TM;
-}
-
-/// Add a new Module to the JIT. If we previously removed the last Module, we
-/// need re-initialize jitstate with a valid Module.
-void JIT::addModule(std::unique_ptr<Module> M) {
- MutexGuard locked(lock);
-
- if (Modules.empty()) {
- assert(!jitstate && "jitstate should be NULL if Modules vector is empty!");
-
- jitstate = new JITState(M.get());
-
- FunctionPassManager &PM = jitstate->getPM();
- M->setDataLayout(TM.getSubtargetImpl()->getDataLayout());
- PM.add(new DataLayoutPass(M.get()));
-
- // Turn the machine code intermediate representation into bytes in memory
- // that may be executed.
- if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) {
- report_fatal_error("Target does not support machine code emission!");
- }
-
- // Initialize passes.
- PM.doInitialization();
- }
-
- ExecutionEngine::addModule(std::move(M));
-}
-
-/// If we are removing the last Module, invalidate the jitstate since the
-/// PassManager it contains references a released Module.
-bool JIT::removeModule(Module *M) {
- bool result = ExecutionEngine::removeModule(M);
-
- MutexGuard locked(lock);
-
- if (jitstate && jitstate->getModule() == M) {
- delete jitstate;
- jitstate = nullptr;
- }
-
- if (!jitstate && !Modules.empty()) {
- jitstate = new JITState(Modules[0].get());
-
- FunctionPassManager &PM = jitstate->getPM();
- M->setDataLayout(TM.getSubtargetImpl()->getDataLayout());
- PM.add(new DataLayoutPass(M));
-
- // Turn the machine code intermediate representation into bytes in memory
- // that may be executed.
- if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) {
- report_fatal_error("Target does not support machine code emission!");
- }
-
- // Initialize passes.
- PM.doInitialization();
- }
- return result;
-}
-
-/// run - Start execution with the specified function and arguments.
-///
-GenericValue JIT::runFunction(Function *F,
- const std::vector<GenericValue> &ArgValues) {
- assert(F && "Function *F was null at entry to run()");
-
- void *FPtr = getPointerToFunction(F);
- assert(FPtr && "Pointer to fn's code was null after getPointerToFunction");
- FunctionType *FTy = F->getFunctionType();
- Type *RetTy = FTy->getReturnType();
-
- assert((FTy->getNumParams() == ArgValues.size() ||
- (FTy->isVarArg() && FTy->getNumParams() <= ArgValues.size())) &&
- "Wrong number of arguments passed into function!");
- assert(FTy->getNumParams() == ArgValues.size() &&
- "This doesn't support passing arguments through varargs (yet)!");
-
- // Handle some common cases first. These cases correspond to common `main'
- // prototypes.
- if (RetTy->isIntegerTy(32) || RetTy->isVoidTy()) {
- switch (ArgValues.size()) {
- case 3:
- if (FTy->getParamType(0)->isIntegerTy(32) &&
- FTy->getParamType(1)->isPointerTy() &&
- FTy->getParamType(2)->isPointerTy()) {
- int (*PF)(int, char **, const char **) =
- (int(*)(int, char **, const char **))(intptr_t)FPtr;
-
- // Call the function.
- GenericValue rv;
- rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(),
- (char **)GVTOP(ArgValues[1]),
- (const char **)GVTOP(ArgValues[2])));
- return rv;
- }
- break;
- case 2:
- if (FTy->getParamType(0)->isIntegerTy(32) &&
- FTy->getParamType(1)->isPointerTy()) {
- int (*PF)(int, char **) = (int(*)(int, char **))(intptr_t)FPtr;
-
- // Call the function.
- GenericValue rv;
- rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(),
- (char **)GVTOP(ArgValues[1])));
- return rv;
- }
- break;
- case 1:
- if (FTy->getParamType(0)->isIntegerTy(32)) {
- GenericValue rv;
- int (*PF)(int) = (int(*)(int))(intptr_t)FPtr;
- rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue()));
- return rv;
- }
- if (FTy->getParamType(0)->isPointerTy()) {
- GenericValue rv;
- int (*PF)(char *) = (int(*)(char *))(intptr_t)FPtr;
- rv.IntVal = APInt(32, PF((char*)GVTOP(ArgValues[0])));
- return rv;
- }
- break;
- }
- }
-
- // Handle cases where no arguments are passed first.
- if (ArgValues.empty()) {
- GenericValue rv;
- switch (RetTy->getTypeID()) {
- default: llvm_unreachable("Unknown return type for function call!");
- case Type::IntegerTyID: {
- unsigned BitWidth = cast<IntegerType>(RetTy)->getBitWidth();
- if (BitWidth == 1)
- rv.IntVal = APInt(BitWidth, ((bool(*)())(intptr_t)FPtr)());
- else if (BitWidth <= 8)
- rv.IntVal = APInt(BitWidth, ((char(*)())(intptr_t)FPtr)());
- else if (BitWidth <= 16)
- rv.IntVal = APInt(BitWidth, ((short(*)())(intptr_t)FPtr)());
- else if (BitWidth <= 32)
- rv.IntVal = APInt(BitWidth, ((int(*)())(intptr_t)FPtr)());
- else if (BitWidth <= 64)
- rv.IntVal = APInt(BitWidth, ((int64_t(*)())(intptr_t)FPtr)());
- else
- llvm_unreachable("Integer types > 64 bits not supported");
- return rv;
- }
- case Type::VoidTyID:
- rv.IntVal = APInt(32, ((int(*)())(intptr_t)FPtr)());
- return rv;
- case Type::FloatTyID:
- rv.FloatVal = ((float(*)())(intptr_t)FPtr)();
- return rv;
- case Type::DoubleTyID:
- rv.DoubleVal = ((double(*)())(intptr_t)FPtr)();
- return rv;
- case Type::X86_FP80TyID:
- case Type::FP128TyID:
- case Type::PPC_FP128TyID:
- llvm_unreachable("long double not supported yet");
- case Type::PointerTyID:
- return PTOGV(((void*(*)())(intptr_t)FPtr)());
- }
- }
-
- // Okay, this is not one of our quick and easy cases. Because we don't have a
- // full FFI, we have to codegen a nullary stub function that just calls the
- // function we are interested in, passing in constants for all of the
- // arguments. Make this function and return.
-
- // First, create the function.
- FunctionType *STy=FunctionType::get(RetTy, false);
- Function *Stub = Function::Create(STy, Function::InternalLinkage, "",
- F->getParent());
-
- // Insert a basic block.
- BasicBlock *StubBB = BasicBlock::Create(F->getContext(), "", Stub);
-
- // Convert all of the GenericValue arguments over to constants. Note that we
- // currently don't support varargs.
- SmallVector<Value*, 8> Args;
- for (unsigned i = 0, e = ArgValues.size(); i != e; ++i) {
- Constant *C = nullptr;
- Type *ArgTy = FTy->getParamType(i);
- const GenericValue &AV = ArgValues[i];
- switch (ArgTy->getTypeID()) {
- default: llvm_unreachable("Unknown argument type for function call!");
- case Type::IntegerTyID:
- C = ConstantInt::get(F->getContext(), AV.IntVal);
- break;
- case Type::FloatTyID:
- C = ConstantFP::get(F->getContext(), APFloat(AV.FloatVal));
- break;
- case Type::DoubleTyID:
- C = ConstantFP::get(F->getContext(), APFloat(AV.DoubleVal));
- break;
- case Type::PPC_FP128TyID:
- case Type::X86_FP80TyID:
- case Type::FP128TyID:
- C = ConstantFP::get(F->getContext(), APFloat(ArgTy->getFltSemantics(),
- AV.IntVal));
- break;
- case Type::PointerTyID:
- void *ArgPtr = GVTOP(AV);
- if (sizeof(void*) == 4)
- C = ConstantInt::get(Type::getInt32Ty(F->getContext()),
- (int)(intptr_t)ArgPtr);
- else
- C = ConstantInt::get(Type::getInt64Ty(F->getContext()),
- (intptr_t)ArgPtr);
- // Cast the integer to pointer
- C = ConstantExpr::getIntToPtr(C, ArgTy);
- break;
- }
- Args.push_back(C);
- }
-
- CallInst *TheCall = CallInst::Create(F, Args, "", StubBB);
- TheCall->setCallingConv(F->getCallingConv());
- TheCall->setTailCall();
- if (!TheCall->getType()->isVoidTy())
- // Return result of the call.
- ReturnInst::Create(F->getContext(), TheCall, StubBB);
- else
- ReturnInst::Create(F->getContext(), StubBB); // Just return void.
-
- // Finally, call our nullary stub function.
- GenericValue Result = runFunction(Stub, std::vector<GenericValue>());
- // Erase it, since no other function can have a reference to it.
- Stub->eraseFromParent();
- // And return the result.
- return Result;
-}
-
-void JIT::RegisterJITEventListener(JITEventListener *L) {
- if (!L)
- return;
- MutexGuard locked(lock);
- EventListeners.push_back(L);
-}
-void JIT::UnregisterJITEventListener(JITEventListener *L) {
- if (!L)
- return;
- MutexGuard locked(lock);
- std::vector<JITEventListener*>::reverse_iterator I=
- std::find(EventListeners.rbegin(), EventListeners.rend(), L);
- if (I != EventListeners.rend()) {
- std::swap(*I, EventListeners.back());
- EventListeners.pop_back();
- }
-}
-void JIT::NotifyFunctionEmitted(
- const Function &F,
- void *Code, size_t Size,
- const JITEvent_EmittedFunctionDetails &Details) {
- MutexGuard locked(lock);
- for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
- EventListeners[I]->NotifyFunctionEmitted(F, Code, Size, Details);
- }
-}
-
-void JIT::NotifyFreeingMachineCode(void *OldPtr) {
- MutexGuard locked(lock);
- for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
- EventListeners[I]->NotifyFreeingMachineCode(OldPtr);
- }
-}
-
-/// runJITOnFunction - Run the FunctionPassManager full of
-/// just-in-time compilation passes on F, hopefully filling in
-/// GlobalAddress[F] with the address of F's machine code.
-///
-void JIT::runJITOnFunction(Function *F, MachineCodeInfo *MCI) {
- MutexGuard locked(lock);
-
- class MCIListener : public JITEventListener {
- MachineCodeInfo *const MCI;
- public:
- MCIListener(MachineCodeInfo *mci) : MCI(mci) {}
- void NotifyFunctionEmitted(const Function &, void *Code, size_t Size,
- const EmittedFunctionDetails &) override {
- MCI->setAddress(Code);
- MCI->setSize(Size);
- }
- };
- MCIListener MCIL(MCI);
- if (MCI)
- RegisterJITEventListener(&MCIL);
-
- runJITOnFunctionUnlocked(F);
-
- if (MCI)
- UnregisterJITEventListener(&MCIL);
-}
-
-void JIT::runJITOnFunctionUnlocked(Function *F) {
- assert(!isAlreadyCodeGenerating && "Error: Recursive compilation detected!");
-
- jitTheFunctionUnlocked(F);
-
- // If the function referred to another function that had not yet been
- // read from bitcode, and we are jitting non-lazily, emit it now.
- while (!jitstate->getPendingFunctions().empty()) {
- Function *PF = jitstate->getPendingFunctions().back();
- jitstate->getPendingFunctions().pop_back();
-
- assert(!PF->hasAvailableExternallyLinkage() &&
- "Externally-defined function should not be in pending list.");
-
- jitTheFunctionUnlocked(PF);
-
- // Now that the function has been jitted, ask the JITEmitter to rewrite
- // the stub with real address of the function.
- updateFunctionStubUnlocked(PF);
- }
-}
-
-void JIT::jitTheFunctionUnlocked(Function *F) {
- isAlreadyCodeGenerating = true;
- jitstate->getPM().run(*F);
- isAlreadyCodeGenerating = false;
-
- // clear basic block addresses after this function is done
- getBasicBlockAddressMap().clear();
-}
-
-/// getPointerToFunction - This method is used to get the address of the
-/// specified function, compiling it if necessary.
-///
-void *JIT::getPointerToFunction(Function *F) {
-
- if (void *Addr = getPointerToGlobalIfAvailable(F))
- return Addr; // Check if function already code gen'd
-
- MutexGuard locked(lock);
-
- // Now that this thread owns the lock, make sure we read in the function if it
- // exists in this Module.
- std::string ErrorMsg;
- if (F->Materialize(&ErrorMsg)) {
- report_fatal_error("Error reading function '" + F->getName()+
- "' from bitcode file: " + ErrorMsg);
- }
-
- // ... and check if another thread has already code gen'd the function.
- if (void *Addr = getPointerToGlobalIfAvailable(F))
- return Addr;
-
- if (F->isDeclaration() || F->hasAvailableExternallyLinkage()) {
- bool AbortOnFailure = !F->hasExternalWeakLinkage();
- void *Addr = getPointerToNamedFunction(F->getName(), AbortOnFailure);
- addGlobalMapping(F, Addr);
- return Addr;
- }
-
- runJITOnFunctionUnlocked(F);
-
- void *Addr = getPointerToGlobalIfAvailable(F);
- assert(Addr && "Code generation didn't add function to GlobalAddress table!");
- return Addr;
-}
-
-void JIT::addPointerToBasicBlock(const BasicBlock *BB, void *Addr) {
- MutexGuard locked(lock);
-
- BasicBlockAddressMapTy::iterator I =
- getBasicBlockAddressMap().find(BB);
- if (I == getBasicBlockAddressMap().end()) {
- getBasicBlockAddressMap()[BB] = Addr;
- } else {
- // ignore repeats: some BBs can be split into few MBBs?
- }
-}
-
-void JIT::clearPointerToBasicBlock(const BasicBlock *BB) {
- MutexGuard locked(lock);
- getBasicBlockAddressMap().erase(BB);
-}
-
-void *JIT::getPointerToBasicBlock(BasicBlock *BB) {
- // make sure it's function is compiled by JIT
- (void)getPointerToFunction(BB->getParent());
-
- // resolve basic block address
- MutexGuard locked(lock);
-
- BasicBlockAddressMapTy::iterator I =
- getBasicBlockAddressMap().find(BB);
- if (I != getBasicBlockAddressMap().end()) {
- return I->second;
- } else {
- llvm_unreachable("JIT does not have BB address for address-of-label, was"
- " it eliminated by optimizer?");
- }
-}
-
-void *JIT::getPointerToNamedFunction(const std::string &Name,
- bool AbortOnFailure){
- if (!isSymbolSearchingDisabled()) {
- void *ptr = JMM->getPointerToNamedFunction(Name, false);
- if (ptr)
- return ptr;
- }
-
- /// If a LazyFunctionCreator is installed, use it to get/create the function.
- if (LazyFunctionCreator)
- if (void *RP = LazyFunctionCreator(Name))
- return RP;
-
- if (AbortOnFailure) {
- report_fatal_error("Program used external function '"+Name+
- "' which could not be resolved!");
- }
- return nullptr;
-}
-
-
-/// getOrEmitGlobalVariable - Return the address of the specified global
-/// variable, possibly emitting it to memory if needed. This is used by the
-/// Emitter.
-void *JIT::getOrEmitGlobalVariable(const GlobalVariable *GV) {
- MutexGuard locked(lock);
-
- void *Ptr = getPointerToGlobalIfAvailable(GV);
- if (Ptr) return Ptr;
-
- // If the global is external, just remember the address.
- if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) {
-#if HAVE___DSO_HANDLE
- if (GV->getName() == "__dso_handle")
- return (void*)&__dso_handle;
-#endif
- Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(GV->getName());
- if (!Ptr) {
- report_fatal_error("Could not resolve external global address: "
- +GV->getName());
- }
- addGlobalMapping(GV, Ptr);
- } else {
- // If the global hasn't been emitted to memory yet, allocate space and
- // emit it into memory.
- Ptr = getMemoryForGV(GV);
- addGlobalMapping(GV, Ptr);
- EmitGlobalVariable(GV); // Initialize the variable.
- }
- return Ptr;
-}
-
-/// recompileAndRelinkFunction - This method is used to force a function
-/// which has already been compiled, to be compiled again, possibly
-/// after it has been modified. Then the entry to the old copy is overwritten
-/// with a branch to the new copy. If there was no old copy, this acts
-/// just like JIT::getPointerToFunction().
-///
-void *JIT::recompileAndRelinkFunction(Function *F) {
- void *OldAddr = getPointerToGlobalIfAvailable(F);
-
- // If it's not already compiled there is no reason to patch it up.
- if (!OldAddr) return getPointerToFunction(F);
-
- // Delete the old function mapping.
- addGlobalMapping(F, nullptr);
-
- // Recodegen the function
- runJITOnFunction(F);
-
- // Update state, forward the old function to the new function.
- void *Addr = getPointerToGlobalIfAvailable(F);
- assert(Addr && "Code generation didn't add function to GlobalAddress table!");
- TJI.replaceMachineCodeForFunction(OldAddr, Addr);
- return Addr;
-}
-
-/// getMemoryForGV - This method abstracts memory allocation of global
-/// variable so that the JIT can allocate thread local variables depending
-/// on the target.
-///
-char* JIT::getMemoryForGV(const GlobalVariable* GV) {
- char *Ptr;
-
- // GlobalVariable's which are not "constant" will cause trouble in a server
- // situation. It's returned in the same block of memory as code which may
- // not be writable.
- if (isGVCompilationDisabled() && !GV->isConstant()) {
- report_fatal_error("Compilation of non-internal GlobalValue is disabled!");
- }
-
- // Some applications require globals and code to live together, so they may
- // be allocated into the same buffer, but in general globals are allocated
- // through the memory manager which puts them near the code but not in the
- // same buffer.
- Type *GlobalType = GV->getType()->getElementType();
- size_t S = getDataLayout()->getTypeAllocSize(GlobalType);
- size_t A = getDataLayout()->getPreferredAlignment(GV);
- if (GV->isThreadLocal()) {
- MutexGuard locked(lock);
- Ptr = TJI.allocateThreadLocalMemory(S);
- } else if (TJI.allocateSeparateGVMemory()) {
- if (A <= 8) {
- Ptr = (char*)malloc(S);
- } else {
- // Allocate S+A bytes of memory, then use an aligned pointer within that
- // space.
- Ptr = (char*)malloc(S+A);
- unsigned MisAligned = ((intptr_t)Ptr & (A-1));
- Ptr = Ptr + (MisAligned ? (A-MisAligned) : 0);
- }
- } else if (AllocateGVsWithCode) {
- Ptr = (char*)JCE->allocateSpace(S, A);
- } else {
- Ptr = (char*)JCE->allocateGlobal(S, A);
- }
- return Ptr;
-}
-
-void JIT::addPendingFunction(Function *F) {
- MutexGuard locked(lock);
- jitstate->getPendingFunctions().push_back(F);
-}
-
-
-JITEventListener::~JITEventListener() {}
+++ /dev/null
-//===-- JIT.h - Class definition for the JIT --------------------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the top-level JIT data structure.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_EXECUTIONENGINE_JIT_JIT_H
-#define LLVM_LIB_EXECUTIONENGINE_JIT_JIT_H
-
-#include "llvm/ExecutionEngine/ExecutionEngine.h"
-#include "llvm/IR/ValueHandle.h"
-#include "llvm/PassManager.h"
-
-namespace llvm {
-
-class Function;
-struct JITEvent_EmittedFunctionDetails;
-class MachineCodeEmitter;
-class MachineCodeInfo;
-class TargetJITInfo;
-class TargetMachine;
-
-class JITState {
-private:
- FunctionPassManager PM; // Passes to compile a function
- Module *M; // Module used to create the PM
-
- /// PendingFunctions - Functions which have not been code generated yet, but
- /// were called from a function being code generated.
- std::vector<AssertingVH<Function> > PendingFunctions;
-
-public:
- explicit JITState(Module *M) : PM(M), M(M) {}
-
- FunctionPassManager &getPM() {
- return PM;
- }
-
- Module *getModule() const { return M; }
- std::vector<AssertingVH<Function> > &getPendingFunctions() {
- return PendingFunctions;
- }
-};
-
-
-class JIT : public ExecutionEngine {
- /// types
- typedef ValueMap<const BasicBlock *, void *>
- BasicBlockAddressMapTy;
- /// data
- TargetMachine &TM; // The current target we are compiling to
- TargetJITInfo &TJI; // The JITInfo for the target we are compiling to
- JITCodeEmitter *JCE; // JCE object
- JITMemoryManager *JMM;
- std::vector<JITEventListener*> EventListeners;
-
- /// AllocateGVsWithCode - Some applications require that global variables and
- /// code be allocated into the same region of memory, in which case this flag
- /// should be set to true. Doing so breaks freeMachineCodeForFunction.
- bool AllocateGVsWithCode;
-
- /// True while the JIT is generating code. Used to assert against recursive
- /// entry.
- bool isAlreadyCodeGenerating;
-
- JITState *jitstate;
-
- /// BasicBlockAddressMap - A mapping between LLVM basic blocks and their
- /// actualized version, only filled for basic blocks that have their address
- /// taken.
- BasicBlockAddressMapTy BasicBlockAddressMap;
-
-
- JIT(std::unique_ptr<Module> M, TargetMachine &tm, TargetJITInfo &tji,
- JITMemoryManager *JMM, bool AllocateGVsWithCode);
-public:
- ~JIT();
-
- static void Register() {
- JITCtor = createJIT;
- }
-
- /// getJITInfo - Return the target JIT information structure.
- ///
- TargetJITInfo &getJITInfo() const { return TJI; }
-
- void addModule(std::unique_ptr<Module> M) override;
-
- /// removeModule - Remove a Module from the list of modules. Returns true if
- /// M is found.
- bool removeModule(Module *M) override;
-
- /// runFunction - Start execution with the specified function and arguments.
- ///
- GenericValue runFunction(Function *F,
- const std::vector<GenericValue> &ArgValues) override;
-
- /// getPointerToNamedFunction - This method returns the address of the
- /// specified function by using the MemoryManager. As such it is only
- /// useful for resolving library symbols, not code generated symbols.
- ///
- /// If AbortOnFailure is false and no function with the given name is
- /// found, this function silently returns a null pointer. Otherwise,
- /// it prints a message to stderr and aborts.
- ///
- void *getPointerToNamedFunction(const std::string &Name,
- bool AbortOnFailure = true) override;
-
- // CompilationCallback - Invoked the first time that a call site is found,
- // which causes lazy compilation of the target function.
- //
- static void CompilationCallback();
-
- /// getPointerToFunction - This returns the address of the specified function,
- /// compiling it if necessary.
- ///
- void *getPointerToFunction(Function *F) override;
-
- /// addPointerToBasicBlock - Adds address of the specific basic block.
- void addPointerToBasicBlock(const BasicBlock *BB, void *Addr);
-
- /// clearPointerToBasicBlock - Removes address of specific basic block.
- void clearPointerToBasicBlock(const BasicBlock *BB);
-
- /// getPointerToBasicBlock - This returns the address of the specified basic
- /// block, assuming function is compiled.
- void *getPointerToBasicBlock(BasicBlock *BB) override;
-
- /// getOrEmitGlobalVariable - Return the address of the specified global
- /// variable, possibly emitting it to memory if needed. This is used by the
- /// Emitter.
- void *getOrEmitGlobalVariable(const GlobalVariable *GV) override;
-
- /// getPointerToFunctionOrStub - If the specified function has been
- /// code-gen'd, return a pointer to the function. If not, compile it, or use
- /// a stub to implement lazy compilation if available.
- ///
- void *getPointerToFunctionOrStub(Function *F) override;
-
- /// recompileAndRelinkFunction - This method is used to force a function
- /// which has already been compiled, to be compiled again, possibly
- /// after it has been modified. Then the entry to the old copy is overwritten
- /// with a branch to the new copy. If there was no old copy, this acts
- /// just like JIT::getPointerToFunction().
- ///
- void *recompileAndRelinkFunction(Function *F) override;
-
- /// freeMachineCodeForFunction - deallocate memory used to code-generate this
- /// Function.
- ///
- void freeMachineCodeForFunction(Function *F) override;
-
- /// addPendingFunction - while jitting non-lazily, a called but non-codegen'd
- /// function was encountered. Add it to a pending list to be processed after
- /// the current function.
- ///
- void addPendingFunction(Function *F);
-
- /// getCodeEmitter - Return the code emitter this JIT is emitting into.
- ///
- JITCodeEmitter *getCodeEmitter() const { return JCE; }
-
- static ExecutionEngine *createJIT(std::unique_ptr<Module> M,
- std::string *ErrorStr,
- JITMemoryManager *JMM,
- bool GVsWithCode,
- TargetMachine *TM);
-
- // Run the JIT on F and return information about the generated code
- void runJITOnFunction(Function *F, MachineCodeInfo *MCI = nullptr) override;
-
- void RegisterJITEventListener(JITEventListener *L) override;
- void UnregisterJITEventListener(JITEventListener *L) override;
-
- TargetMachine *getTargetMachine() override { return &TM; }
-
- /// These functions correspond to the methods on JITEventListener. They
- /// iterate over the registered listeners and call the corresponding method on
- /// each.
- void NotifyFunctionEmitted(
- const Function &F, void *Code, size_t Size,
- const JITEvent_EmittedFunctionDetails &Details);
- void NotifyFreeingMachineCode(void *OldPtr);
-
- BasicBlockAddressMapTy &
- getBasicBlockAddressMap() {
- return BasicBlockAddressMap;
- }
-
-
-private:
- static JITCodeEmitter *createEmitter(JIT &J, JITMemoryManager *JMM,
- TargetMachine &tm);
- void runJITOnFunctionUnlocked(Function *F);
- void updateFunctionStubUnlocked(Function *F);
- void jitTheFunctionUnlocked(Function *F);
-
-protected:
-
- /// getMemoryforGV - Allocate memory for a global variable.
- char* getMemoryForGV(const GlobalVariable* GV) override;
-
-};
-
-} // End llvm namespace
-
-#endif
+++ /dev/null
-//===-- JITEmitter.cpp - Write machine code to executable memory ----------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines a MachineCodeEmitter object that is used by the JIT to
-// write machine code to memory and remember where relocatable values are.
-//
-//===----------------------------------------------------------------------===//
-
-#include "JIT.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/CodeGen/JITCodeEmitter.h"
-#include "llvm/CodeGen/MachineCodeInfo.h"
-#include "llvm/CodeGen/MachineConstantPool.h"
-#include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineJumpTableInfo.h"
-#include "llvm/CodeGen/MachineModuleInfo.h"
-#include "llvm/CodeGen/MachineRelocation.h"
-#include "llvm/ExecutionEngine/GenericValue.h"
-#include "llvm/ExecutionEngine/JITEventListener.h"
-#include "llvm/ExecutionEngine/JITMemoryManager.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/DebugInfo.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Operator.h"
-#include "llvm/IR/ValueHandle.h"
-#include "llvm/IR/ValueMap.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/ManagedStatic.h"
-#include "llvm/Support/Memory.h"
-#include "llvm/Support/MutexGuard.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetInstrInfo.h"
-#include "llvm/Target/TargetJITInfo.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetOptions.h"
-#include <algorithm>
-#ifndef NDEBUG
-#include <iomanip>
-#endif
-using namespace llvm;
-
-#define DEBUG_TYPE "jit"
-
-STATISTIC(NumBytes, "Number of bytes of machine code compiled");
-STATISTIC(NumRelos, "Number of relocations applied");
-STATISTIC(NumRetries, "Number of retries with more memory");
-
-
-// A declaration may stop being a declaration once it's fully read from bitcode.
-// This function returns true if F is fully read and is still a declaration.
-static bool isNonGhostDeclaration(const Function *F) {
- return F->isDeclaration() && !F->isMaterializable();
-}
-
-//===----------------------------------------------------------------------===//
-// JIT lazy compilation code.
-//
-namespace {
- class JITEmitter;
- class JITResolverState;
-
- template<typename ValueTy>
- struct NoRAUWValueMapConfig : public ValueMapConfig<ValueTy> {
- typedef JITResolverState *ExtraData;
- static void onRAUW(JITResolverState *, Value *Old, Value *New) {
- llvm_unreachable("The JIT doesn't know how to handle a"
- " RAUW on a value it has emitted.");
- }
- };
-
- struct CallSiteValueMapConfig : public NoRAUWValueMapConfig<Function*> {
- typedef JITResolverState *ExtraData;
- static void onDelete(JITResolverState *JRS, Function *F);
- };
-
- class JITResolverState {
- public:
- typedef ValueMap<Function*, void*, NoRAUWValueMapConfig<Function*> >
- FunctionToLazyStubMapTy;
- typedef std::map<void*, AssertingVH<Function> > CallSiteToFunctionMapTy;
- typedef ValueMap<Function *, SmallPtrSet<void*, 1>,
- CallSiteValueMapConfig> FunctionToCallSitesMapTy;
- typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy;
- private:
- /// FunctionToLazyStubMap - Keep track of the lazy stub created for a
- /// particular function so that we can reuse them if necessary.
- FunctionToLazyStubMapTy FunctionToLazyStubMap;
-
- /// CallSiteToFunctionMap - Keep track of the function that each lazy call
- /// site corresponds to, and vice versa.
- CallSiteToFunctionMapTy CallSiteToFunctionMap;
- FunctionToCallSitesMapTy FunctionToCallSitesMap;
-
- /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a
- /// particular GlobalVariable so that we can reuse them if necessary.
- GlobalToIndirectSymMapTy GlobalToIndirectSymMap;
-
-#ifndef NDEBUG
- /// Instance of the JIT this ResolverState serves.
- JIT *TheJIT;
-#endif
-
- public:
- JITResolverState(JIT *jit) : FunctionToLazyStubMap(this),
- FunctionToCallSitesMap(this) {
-#ifndef NDEBUG
- TheJIT = jit;
-#endif
- }
-
- FunctionToLazyStubMapTy& getFunctionToLazyStubMap() {
- return FunctionToLazyStubMap;
- }
-
- GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap() {
- return GlobalToIndirectSymMap;
- }
-
- std::pair<void *, Function *> LookupFunctionFromCallSite(
- void *CallSite) const {
- // The address given to us for the stub may not be exactly right, it
- // might be a little bit after the stub. As such, use upper_bound to
- // find it.
- CallSiteToFunctionMapTy::const_iterator I =
- CallSiteToFunctionMap.upper_bound(CallSite);
- assert(I != CallSiteToFunctionMap.begin() &&
- "This is not a known call site!");
- --I;
- return *I;
- }
-
- void AddCallSite(void *CallSite, Function *F) {
- bool Inserted = CallSiteToFunctionMap.insert(
- std::make_pair(CallSite, F)).second;
- (void)Inserted;
- assert(Inserted && "Pair was already in CallSiteToFunctionMap");
- FunctionToCallSitesMap[F].insert(CallSite);
- }
-
- void EraseAllCallSitesForPrelocked(Function *F);
-
- // Erases _all_ call sites regardless of their function. This is used to
- // unregister the stub addresses from the StubToResolverMap in
- // ~JITResolver().
- void EraseAllCallSitesPrelocked();
- };
-
- /// JITResolver - Keep track of, and resolve, call sites for functions that
- /// have not yet been compiled.
- class JITResolver {
- typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy;
- typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy;
- typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy;
-
- /// LazyResolverFn - The target lazy resolver function that we actually
- /// rewrite instructions to use.
- TargetJITInfo::LazyResolverFn LazyResolverFn;
-
- JITResolverState state;
-
- /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap
- /// for external functions. TODO: Of course, external functions don't need
- /// a lazy stub. It's actually here to make it more likely that far calls
- /// succeed, but no single stub can guarantee that. I'll remove this in a
- /// subsequent checkin when I actually fix far calls.
- std::map<void*, void*> ExternalFnToStubMap;
-
- /// revGOTMap - map addresses to indexes in the GOT
- std::map<void*, unsigned> revGOTMap;
- unsigned nextGOTIndex;
-
- JITEmitter &JE;
-
- /// Instance of JIT corresponding to this Resolver.
- JIT *TheJIT;
-
- public:
- explicit JITResolver(JIT &jit, JITEmitter &je)
- : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) {
- LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn);
- }
-
- ~JITResolver();
-
- /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's
- /// lazy-compilation stub if it has already been created.
- void *getLazyFunctionStubIfAvailable(Function *F);
-
- /// getLazyFunctionStub - This returns a pointer to a function's
- /// lazy-compilation stub, creating one on demand as needed.
- void *getLazyFunctionStub(Function *F);
-
- /// getExternalFunctionStub - Return a stub for the function at the
- /// specified address, created lazily on demand.
- void *getExternalFunctionStub(void *FnAddr);
-
- /// getGlobalValueIndirectSym - Return an indirect symbol containing the
- /// specified GV address.
- void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress);
-
- /// getGOTIndexForAddress - Return a new or existing index in the GOT for
- /// an address. This function only manages slots, it does not manage the
- /// contents of the slots or the memory associated with the GOT.
- unsigned getGOTIndexForAddr(void *addr);
-
- /// JITCompilerFn - This function is called to resolve a stub to a compiled
- /// address. If the LLVM Function corresponding to the stub has not yet
- /// been compiled, this function compiles it first.
- static void *JITCompilerFn(void *Stub);
- };
-
- class StubToResolverMapTy {
- /// Map a stub address to a specific instance of a JITResolver so that
- /// lazily-compiled functions can find the right resolver to use.
- ///
- /// Guarded by Lock.
- std::map<void*, JITResolver*> Map;
-
- /// Guards Map from concurrent accesses.
- mutable sys::Mutex Lock;
-
- public:
- /// Registers a Stub to be resolved by Resolver.
- void RegisterStubResolver(void *Stub, JITResolver *Resolver) {
- MutexGuard guard(Lock);
- Map.insert(std::make_pair(Stub, Resolver));
- }
- /// Unregisters the Stub when it's invalidated.
- void UnregisterStubResolver(void *Stub) {
- MutexGuard guard(Lock);
- Map.erase(Stub);
- }
- /// Returns the JITResolver instance that owns the Stub.
- JITResolver *getResolverFromStub(void *Stub) const {
- MutexGuard guard(Lock);
- // The address given to us for the stub may not be exactly right, it might
- // be a little bit after the stub. As such, use upper_bound to find it.
- // This is the same trick as in LookupFunctionFromCallSite from
- // JITResolverState.
- std::map<void*, JITResolver*>::const_iterator I = Map.upper_bound(Stub);
- assert(I != Map.begin() && "This is not a known stub!");
- --I;
- return I->second;
- }
- /// True if any stubs refer to the given resolver. Only used in an assert().
- /// O(N)
- bool ResolverHasStubs(JITResolver* Resolver) const {
- MutexGuard guard(Lock);
- for (std::map<void*, JITResolver*>::const_iterator I = Map.begin(),
- E = Map.end(); I != E; ++I) {
- if (I->second == Resolver)
- return true;
- }
- return false;
- }
- };
- /// This needs to be static so that a lazy call stub can access it with no
- /// context except the address of the stub.
- ManagedStatic<StubToResolverMapTy> StubToResolverMap;
-
- /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is
- /// used to output functions to memory for execution.
- class JITEmitter : public JITCodeEmitter {
- JITMemoryManager *MemMgr;
-
- // When outputting a function stub in the context of some other function, we
- // save BufferBegin/BufferEnd/CurBufferPtr here.
- uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr;
-
- // When reattempting to JIT a function after running out of space, we store
- // the estimated size of the function we're trying to JIT here, so we can
- // ask the memory manager for at least this much space. When we
- // successfully emit the function, we reset this back to zero.
- uintptr_t SizeEstimate;
-
- /// Relocations - These are the relocations that the function needs, as
- /// emitted.
- std::vector<MachineRelocation> Relocations;
-
- /// MBBLocations - This vector is a mapping from MBB ID's to their address.
- /// It is filled in by the StartMachineBasicBlock callback and queried by
- /// the getMachineBasicBlockAddress callback.
- std::vector<uintptr_t> MBBLocations;
-
- /// ConstantPool - The constant pool for the current function.
- ///
- MachineConstantPool *ConstantPool;
-
- /// ConstantPoolBase - A pointer to the first entry in the constant pool.
- ///
- void *ConstantPoolBase;
-
- /// ConstPoolAddresses - Addresses of individual constant pool entries.
- ///
- SmallVector<uintptr_t, 8> ConstPoolAddresses;
-
- /// JumpTable - The jump tables for the current function.
- ///
- MachineJumpTableInfo *JumpTable;
-
- /// JumpTableBase - A pointer to the first entry in the jump table.
- ///
- void *JumpTableBase;
-
- /// Resolver - This contains info about the currently resolved functions.
- JITResolver Resolver;
-
- /// LabelLocations - This vector is a mapping from Label ID's to their
- /// address.
- DenseMap<MCSymbol*, uintptr_t> LabelLocations;
-
- /// MMI - Machine module info for exception informations
- MachineModuleInfo* MMI;
-
- // CurFn - The llvm function being emitted. Only valid during
- // finishFunction().
- const Function *CurFn;
-
- /// Information about emitted code, which is passed to the
- /// JITEventListeners. This is reset in startFunction and used in
- /// finishFunction.
- JITEvent_EmittedFunctionDetails EmissionDetails;
-
- struct EmittedCode {
- void *FunctionBody; // Beginning of the function's allocation.
- void *Code; // The address the function's code actually starts at.
- void *ExceptionTable;
- EmittedCode() : FunctionBody(nullptr), Code(nullptr),
- ExceptionTable(nullptr) {}
- };
- struct EmittedFunctionConfig : public ValueMapConfig<const Function*> {
- typedef JITEmitter *ExtraData;
- static void onDelete(JITEmitter *, const Function*);
- static void onRAUW(JITEmitter *, const Function*, const Function*);
- };
- ValueMap<const Function *, EmittedCode,
- EmittedFunctionConfig> EmittedFunctions;
-
- DebugLoc PrevDL;
-
- /// Instance of the JIT
- JIT *TheJIT;
-
- public:
- JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM)
- : SizeEstimate(0), Resolver(jit, *this), MMI(nullptr), CurFn(nullptr),
- EmittedFunctions(this), TheJIT(&jit) {
- MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager();
- if (jit.getJITInfo().needsGOT()) {
- MemMgr->AllocateGOT();
- DEBUG(dbgs() << "JIT is managing a GOT\n");
- }
-
- }
- ~JITEmitter() {
- delete MemMgr;
- }
-
- JITResolver &getJITResolver() { return Resolver; }
-
- void startFunction(MachineFunction &F) override;
- bool finishFunction(MachineFunction &F) override;
-
- void emitConstantPool(MachineConstantPool *MCP);
- void initJumpTableInfo(MachineJumpTableInfo *MJTI);
- void emitJumpTableInfo(MachineJumpTableInfo *MJTI);
-
- void startGVStub(const GlobalValue* GV,
- unsigned StubSize, unsigned Alignment = 1);
- void startGVStub(void *Buffer, unsigned StubSize);
- void finishGVStub();
- void *allocIndirectGV(const GlobalValue *GV, const uint8_t *Buffer,
- size_t Size, unsigned Alignment) override;
-
- /// allocateSpace - Reserves space in the current block if any, or
- /// allocate a new one of the given size.
- void *allocateSpace(uintptr_t Size, unsigned Alignment) override;
-
- /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace,
- /// this method does not allocate memory in the current output buffer,
- /// because a global may live longer than the current function.
- void *allocateGlobal(uintptr_t Size, unsigned Alignment) override;
-
- void addRelocation(const MachineRelocation &MR) override {
- Relocations.push_back(MR);
- }
-
- void StartMachineBasicBlock(MachineBasicBlock *MBB) override {
- if (MBBLocations.size() <= (unsigned)MBB->getNumber())
- MBBLocations.resize((MBB->getNumber()+1)*2);
- MBBLocations[MBB->getNumber()] = getCurrentPCValue();
- if (MBB->hasAddressTaken())
- TheJIT->addPointerToBasicBlock(MBB->getBasicBlock(),
- (void*)getCurrentPCValue());
- DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at ["
- << (void*) getCurrentPCValue() << "]\n");
- }
-
- uintptr_t getConstantPoolEntryAddress(unsigned Entry) const override;
- uintptr_t getJumpTableEntryAddress(unsigned Entry) const override;
-
- uintptr_t
- getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override {
- assert(MBBLocations.size() > (unsigned)MBB->getNumber() &&
- MBBLocations[MBB->getNumber()] && "MBB not emitted!");
- return MBBLocations[MBB->getNumber()];
- }
-
- /// retryWithMoreMemory - Log a retry and deallocate all memory for the
- /// given function. Increase the minimum allocation size so that we get
- /// more memory next time.
- void retryWithMoreMemory(MachineFunction &F);
-
- /// deallocateMemForFunction - Deallocate all memory for the specified
- /// function body.
- void deallocateMemForFunction(const Function *F);
-
- void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) override;
-
- void emitLabel(MCSymbol *Label) override {
- LabelLocations[Label] = getCurrentPCValue();
- }
-
- DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() override {
- return &LabelLocations;
- }
-
- uintptr_t getLabelAddress(MCSymbol *Label) const override {
- assert(LabelLocations.count(Label) && "Label not emitted!");
- return LabelLocations.find(Label)->second;
- }
-
- void setModuleInfo(MachineModuleInfo* Info) override {
- MMI = Info;
- }
-
- private:
- void *getPointerToGlobal(GlobalValue *GV, void *Reference,
- bool MayNeedFarStub);
- void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference);
- };
-}
-
-void CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) {
- JRS->EraseAllCallSitesForPrelocked(F);
-}
-
-void JITResolverState::EraseAllCallSitesForPrelocked(Function *F) {
- FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F);
- if (F2C == FunctionToCallSitesMap.end())
- return;
- StubToResolverMapTy &S2RMap = *StubToResolverMap;
- for (void *C : F2C->second) {
- S2RMap.UnregisterStubResolver(C);
- bool Erased = CallSiteToFunctionMap.erase(C);
- (void)Erased;
- assert(Erased && "Missing call site->function mapping");
- }
- FunctionToCallSitesMap.erase(F2C);
-}
-
-void JITResolverState::EraseAllCallSitesPrelocked() {
- StubToResolverMapTy &S2RMap = *StubToResolverMap;
- for (CallSiteToFunctionMapTy::const_iterator
- I = CallSiteToFunctionMap.begin(),
- E = CallSiteToFunctionMap.end(); I != E; ++I) {
- S2RMap.UnregisterStubResolver(I->first);
- }
- CallSiteToFunctionMap.clear();
- FunctionToCallSitesMap.clear();
-}
-
-JITResolver::~JITResolver() {
- // No need to lock because we're in the destructor, and state isn't shared.
- state.EraseAllCallSitesPrelocked();
- assert(!StubToResolverMap->ResolverHasStubs(this) &&
- "Resolver destroyed with stubs still alive.");
-}
-
-/// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub
-/// if it has already been created.
-void *JITResolver::getLazyFunctionStubIfAvailable(Function *F) {
- MutexGuard locked(TheJIT->lock);
-
- // If we already have a stub for this function, recycle it.
- return state.getFunctionToLazyStubMap().lookup(F);
-}
-
-/// getFunctionStub - This returns a pointer to a function stub, creating
-/// one on demand as needed.
-void *JITResolver::getLazyFunctionStub(Function *F) {
- MutexGuard locked(TheJIT->lock);
-
- // If we already have a lazy stub for this function, recycle it.
- void *&Stub = state.getFunctionToLazyStubMap()[F];
- if (Stub) return Stub;
-
- // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we
- // must resolve the symbol now.
- void *Actual = TheJIT->isCompilingLazily()
- ? (void *)(intptr_t)LazyResolverFn : (void *)nullptr;
-
- // If this is an external declaration, attempt to resolve the address now
- // to place in the stub.
- if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) {
- Actual = TheJIT->getPointerToFunction(F);
-
- // If we resolved the symbol to a null address (eg. a weak external)
- // don't emit a stub. Return a null pointer to the application.
- if (!Actual) return nullptr;
- }
-
- TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout();
- JE.startGVStub(F, SL.Size, SL.Alignment);
- // Codegen a new stub, calling the lazy resolver or the actual address of the
- // external function, if it was resolved.
- Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE);
- JE.finishGVStub();
-
- if (Actual != (void*)(intptr_t)LazyResolverFn) {
- // If we are getting the stub for an external function, we really want the
- // address of the stub in the GlobalAddressMap for the JIT, not the address
- // of the external function.
- TheJIT->updateGlobalMapping(F, Stub);
- }
-
- DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '"
- << F->getName() << "'\n");
-
- if (TheJIT->isCompilingLazily()) {
- // Register this JITResolver as the one corresponding to this call site so
- // JITCompilerFn will be able to find it.
- StubToResolverMap->RegisterStubResolver(Stub, this);
-
- // Finally, keep track of the stub-to-Function mapping so that the
- // JITCompilerFn knows which function to compile!
- state.AddCallSite(Stub, F);
- } else if (!Actual) {
- // If we are JIT'ing non-lazily but need to call a function that does not
- // exist yet, add it to the JIT's work list so that we can fill in the
- // stub address later.
- assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() &&
- "'Actual' should have been set above.");
- TheJIT->addPendingFunction(F);
- }
-
- return Stub;
-}
-
-/// getGlobalValueIndirectSym - Return a lazy pointer containing the specified
-/// GV address.
-void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) {
- MutexGuard locked(TheJIT->lock);
-
- // If we already have a stub for this global variable, recycle it.
- void *&IndirectSym = state.getGlobalToIndirectSymMap()[GV];
- if (IndirectSym) return IndirectSym;
-
- // Otherwise, codegen a new indirect symbol.
- IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress,
- JE);
-
- DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym
- << "] for GV '" << GV->getName() << "'\n");
-
- return IndirectSym;
-}
-
-/// getExternalFunctionStub - Return a stub for the function at the
-/// specified address, created lazily on demand.
-void *JITResolver::getExternalFunctionStub(void *FnAddr) {
- // If we already have a stub for this function, recycle it.
- void *&Stub = ExternalFnToStubMap[FnAddr];
- if (Stub) return Stub;
-
- TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout();
- JE.startGVStub(nullptr, SL.Size, SL.Alignment);
- Stub = TheJIT->getJITInfo().emitFunctionStub(nullptr, FnAddr, JE);
- JE.finishGVStub();
-
- DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub
- << "] for external function at '" << FnAddr << "'\n");
- return Stub;
-}
-
-unsigned JITResolver::getGOTIndexForAddr(void* addr) {
- unsigned idx = revGOTMap[addr];
- if (!idx) {
- idx = ++nextGOTIndex;
- revGOTMap[addr] = idx;
- DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr ["
- << addr << "]\n");
- }
- return idx;
-}
-
-/// JITCompilerFn - This function is called when a lazy compilation stub has
-/// been entered. It looks up which function this stub corresponds to, compiles
-/// it if necessary, then returns the resultant function pointer.
-void *JITResolver::JITCompilerFn(void *Stub) {
- JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub);
- assert(JR && "Unable to find the corresponding JITResolver to the call site");
-
- Function* F = nullptr;
- void* ActualPtr = nullptr;
-
- {
- // Only lock for getting the Function. The call getPointerToFunction made
- // in this function might trigger function materializing, which requires
- // JIT lock to be unlocked.
- MutexGuard locked(JR->TheJIT->lock);
-
- // The address given to us for the stub may not be exactly right, it might
- // be a little bit after the stub. As such, use upper_bound to find it.
- std::pair<void*, Function*> I =
- JR->state.LookupFunctionFromCallSite(Stub);
- F = I.second;
- ActualPtr = I.first;
- }
-
- // If we have already code generated the function, just return the address.
- void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F);
-
- if (!Result) {
- // Otherwise we don't have it, do lazy compilation now.
-
- // If lazy compilation is disabled, emit a useful error message and abort.
- if (!JR->TheJIT->isCompilingLazily()) {
- report_fatal_error("LLVM JIT requested to do lazy compilation of"
- " function '"
- + F->getName() + "' when lazy compiles are disabled!");
- }
-
- DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName()
- << "' In stub ptr = " << Stub << " actual ptr = "
- << ActualPtr << "\n");
- (void)ActualPtr;
-
- Result = JR->TheJIT->getPointerToFunction(F);
- }
-
- // Reacquire the lock to update the GOT map.
- MutexGuard locked(JR->TheJIT->lock);
-
- // We might like to remove the call site from the CallSiteToFunction map, but
- // we can't do that! Multiple threads could be stuck, waiting to acquire the
- // lock above. As soon as the 1st function finishes compiling the function,
- // the next one will be released, and needs to be able to find the function it
- // needs to call.
-
- // FIXME: We could rewrite all references to this stub if we knew them.
-
- // What we will do is set the compiled function address to map to the
- // same GOT entry as the stub so that later clients may update the GOT
- // if they see it still using the stub address.
- // Note: this is done so the Resolver doesn't have to manage GOT memory
- // Do this without allocating map space if the target isn't using a GOT
- if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end())
- JR->revGOTMap[Result] = JR->revGOTMap[Stub];
-
- return Result;
-}
-
-//===----------------------------------------------------------------------===//
-// JITEmitter code.
-//
-
-static GlobalObject *getSimpleAliasee(Constant *C) {
- C = C->stripPointerCasts();
- return dyn_cast<GlobalObject>(C);
-}
-
-void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference,
- bool MayNeedFarStub) {
- if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
- return TheJIT->getOrEmitGlobalVariable(GV);
-
- if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) {
- // We can only handle simple cases.
- if (GlobalValue *GV = getSimpleAliasee(GA->getAliasee()))
- return TheJIT->getPointerToGlobal(GV);
- return nullptr;
- }
-
- // If we have already compiled the function, return a pointer to its body.
- Function *F = cast<Function>(V);
-
- void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F);
- if (FnStub) {
- // Return the function stub if it's already created. We do this first so
- // that we're returning the same address for the function as any previous
- // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be
- // close enough to call.
- return FnStub;
- }
-
- // If we know the target can handle arbitrary-distance calls, try to
- // return a direct pointer.
- if (!MayNeedFarStub) {
- // If we have code, go ahead and return that.
- void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F);
- if (ResultPtr) return ResultPtr;
-
- // If this is an external function pointer, we can force the JIT to
- // 'compile' it, which really just adds it to the map.
- if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage())
- return TheJIT->getPointerToFunction(F);
- }
-
- // Otherwise, we may need a to emit a stub, and, conservatively, we always do
- // so. Note that it's possible to return null from getLazyFunctionStub in the
- // case of a weak extern that fails to resolve.
- return Resolver.getLazyFunctionStub(F);
-}
-
-void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) {
- // Make sure GV is emitted first, and create a stub containing the fully
- // resolved address.
- void *GVAddress = getPointerToGlobal(V, Reference, false);
- void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress);
- return StubAddr;
-}
-
-void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) {
- if (DL.isUnknown()) return;
- if (!BeforePrintingInsn) return;
-
- const LLVMContext &Context = EmissionDetails.MF->getFunction()->getContext();
-
- if (DL.getScope(Context) != nullptr && PrevDL != DL) {
- JITEvent_EmittedFunctionDetails::LineStart NextLine;
- NextLine.Address = getCurrentPCValue();
- NextLine.Loc = DL;
- EmissionDetails.LineStarts.push_back(NextLine);
- }
-
- PrevDL = DL;
-}
-
-static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP,
- const DataLayout *TD) {
- const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants();
- if (Constants.empty()) return 0;
-
- unsigned Size = 0;
- for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
- MachineConstantPoolEntry CPE = Constants[i];
- unsigned AlignMask = CPE.getAlignment() - 1;
- Size = (Size + AlignMask) & ~AlignMask;
- Type *Ty = CPE.getType();
- Size += TD->getTypeAllocSize(Ty);
- }
- return Size;
-}
-
-void JITEmitter::startFunction(MachineFunction &F) {
- DEBUG(dbgs() << "JIT: Starting CodeGen of Function "
- << F.getName() << "\n");
-
- uintptr_t ActualSize = 0;
- // Set the memory writable, if it's not already
- MemMgr->setMemoryWritable();
-
- if (SizeEstimate > 0) {
- // SizeEstimate will be non-zero on reallocation attempts.
- ActualSize = SizeEstimate;
- }
-
- BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(),
- ActualSize);
- BufferEnd = BufferBegin+ActualSize;
- EmittedFunctions[F.getFunction()].FunctionBody = BufferBegin;
-
- // Ensure the constant pool/jump table info is at least 4-byte aligned.
- emitAlignment(16);
-
- emitConstantPool(F.getConstantPool());
- if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo())
- initJumpTableInfo(MJTI);
-
- // About to start emitting the machine code for the function.
- emitAlignment(std::max(F.getFunction()->getAlignment(), 8U));
- TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr);
- EmittedFunctions[F.getFunction()].Code = CurBufferPtr;
-
- MBBLocations.clear();
-
- EmissionDetails.MF = &F;
- EmissionDetails.LineStarts.clear();
-}
-
-bool JITEmitter::finishFunction(MachineFunction &F) {
- if (CurBufferPtr == BufferEnd) {
- // We must call endFunctionBody before retrying, because
- // deallocateMemForFunction requires it.
- MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr);
- retryWithMoreMemory(F);
- return true;
- }
-
- if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo())
- emitJumpTableInfo(MJTI);
-
- // FnStart is the start of the text, not the start of the constant pool and
- // other per-function data.
- uint8_t *FnStart =
- (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction());
-
- // FnEnd is the end of the function's machine code.
- uint8_t *FnEnd = CurBufferPtr;
-
- if (!Relocations.empty()) {
- CurFn = F.getFunction();
- NumRelos += Relocations.size();
-
- // Resolve the relocations to concrete pointers.
- for (unsigned i = 0, e = Relocations.size(); i != e; ++i) {
- MachineRelocation &MR = Relocations[i];
- void *ResultPtr = nullptr;
- if (!MR.letTargetResolve()) {
- if (MR.isExternalSymbol()) {
- ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(),
- false);
- DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to ["
- << ResultPtr << "]\n");
-
- // If the target REALLY wants a stub for this function, emit it now.
- if (MR.mayNeedFarStub()) {
- ResultPtr = Resolver.getExternalFunctionStub(ResultPtr);
- }
- } else if (MR.isGlobalValue()) {
- ResultPtr = getPointerToGlobal(MR.getGlobalValue(),
- BufferBegin+MR.getMachineCodeOffset(),
- MR.mayNeedFarStub());
- } else if (MR.isIndirectSymbol()) {
- ResultPtr = getPointerToGVIndirectSym(
- MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset());
- } else if (MR.isBasicBlock()) {
- ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock());
- } else if (MR.isConstantPoolIndex()) {
- ResultPtr =
- (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex());
- } else {
- assert(MR.isJumpTableIndex());
- ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex());
- }
-
- MR.setResultPointer(ResultPtr);
- }
-
- // if we are managing the GOT and the relocation wants an index,
- // give it one
- if (MR.isGOTRelative() && MemMgr->isManagingGOT()) {
- unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr);
- MR.setGOTIndex(idx);
- if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) {
- DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr
- << " pointing at " << ((void**)MemMgr->getGOTBase())[idx]
- << "\n");
- ((void**)MemMgr->getGOTBase())[idx] = ResultPtr;
- }
- }
- }
-
- CurFn = nullptr;
- TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0],
- Relocations.size(), MemMgr->getGOTBase());
- }
-
- // Update the GOT entry for F to point to the new code.
- if (MemMgr->isManagingGOT()) {
- unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin);
- if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) {
- DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin
- << " pointing at " << ((void**)MemMgr->getGOTBase())[idx]
- << "\n");
- ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin;
- }
- }
-
- // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for
- // global variables that were referenced in the relocations.
- MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr);
-
- if (CurBufferPtr == BufferEnd) {
- retryWithMoreMemory(F);
- return true;
- } else {
- // Now that we've succeeded in emitting the function, reset the
- // SizeEstimate back down to zero.
- SizeEstimate = 0;
- }
-
- BufferBegin = CurBufferPtr = nullptr;
- NumBytes += FnEnd-FnStart;
-
- // Invalidate the icache if necessary.
- sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart);
-
- TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart,
- EmissionDetails);
-
- // Reset the previous debug location.
- PrevDL = DebugLoc();
-
- DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart
- << "] Function: " << F.getName()
- << ": " << (FnEnd-FnStart) << " bytes of text, "
- << Relocations.size() << " relocations\n");
-
- Relocations.clear();
- ConstPoolAddresses.clear();
-
- // Mark code region readable and executable if it's not so already.
- MemMgr->setMemoryExecutable();
-
- DEBUG({
- dbgs() << "JIT: Binary code:\n";
- uint8_t* q = FnStart;
- for (int i = 0; q < FnEnd; q += 4, ++i) {
- if (i == 4)
- i = 0;
- if (i == 0)
- dbgs() << "JIT: " << (long)(q - FnStart) << ": ";
- bool Done = false;
- for (int j = 3; j >= 0; --j) {
- if (q + j >= FnEnd)
- Done = true;
- else
- dbgs() << (unsigned short)q[j];
- }
- if (Done)
- break;
- dbgs() << ' ';
- if (i == 3)
- dbgs() << '\n';
- }
- dbgs()<< '\n';
- });
-
- if (MMI)
- MMI->EndFunction();
-
- return false;
-}
-
-void JITEmitter::retryWithMoreMemory(MachineFunction &F) {
- DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n");
- Relocations.clear(); // Clear the old relocations or we'll reapply them.
- ConstPoolAddresses.clear();
- ++NumRetries;
- deallocateMemForFunction(F.getFunction());
- // Try again with at least twice as much free space.
- SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin));
-
- for (MachineFunction::iterator MBB = F.begin(), E = F.end(); MBB != E; ++MBB){
- if (MBB->hasAddressTaken())
- TheJIT->clearPointerToBasicBlock(MBB->getBasicBlock());
- }
-}
-
-/// deallocateMemForFunction - Deallocate all memory for the specified
-/// function body. Also drop any references the function has to stubs.
-/// May be called while the Function is being destroyed inside ~Value().
-void JITEmitter::deallocateMemForFunction(const Function *F) {
- ValueMap<const Function *, EmittedCode, EmittedFunctionConfig>::iterator
- Emitted = EmittedFunctions.find(F);
- if (Emitted != EmittedFunctions.end()) {
- MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody);
- TheJIT->NotifyFreeingMachineCode(Emitted->second.Code);
-
- EmittedFunctions.erase(Emitted);
- }
-}
-
-
-void *JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) {
- if (BufferBegin)
- return JITCodeEmitter::allocateSpace(Size, Alignment);
-
- // create a new memory block if there is no active one.
- // care must be taken so that BufferBegin is invalidated when a
- // block is trimmed
- BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment);
- BufferEnd = BufferBegin+Size;
- return CurBufferPtr;
-}
-
-void *JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) {
- // Delegate this call through the memory manager.
- return MemMgr->allocateGlobal(Size, Alignment);
-}
-
-void JITEmitter::emitConstantPool(MachineConstantPool *MCP) {
- if (TheJIT->getJITInfo().hasCustomConstantPool())
- return;
-
- const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants();
- if (Constants.empty()) return;
-
- unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getDataLayout());
- unsigned Align = MCP->getConstantPoolAlignment();
- ConstantPoolBase = allocateSpace(Size, Align);
- ConstantPool = MCP;
-
- if (!ConstantPoolBase) return; // Buffer overflow.
-
- DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase
- << "] (size: " << Size << ", alignment: " << Align << ")\n");
-
- // Initialize the memory for all of the constant pool entries.
- unsigned Offset = 0;
- for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
- MachineConstantPoolEntry CPE = Constants[i];
- unsigned AlignMask = CPE.getAlignment() - 1;
- Offset = (Offset + AlignMask) & ~AlignMask;
-
- uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset;
- ConstPoolAddresses.push_back(CAddr);
- if (CPE.isMachineConstantPoolEntry()) {
- // FIXME: add support to lower machine constant pool values into bytes!
- report_fatal_error("Initialize memory with machine specific constant pool"
- "entry has not been implemented!");
- }
- TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr);
- DEBUG(dbgs() << "JIT: CP" << i << " at [0x";
- dbgs().write_hex(CAddr) << "]\n");
-
- Type *Ty = CPE.Val.ConstVal->getType();
- Offset += TheJIT->getDataLayout()->getTypeAllocSize(Ty);
- }
-}
-
-void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) {
- if (TheJIT->getJITInfo().hasCustomJumpTables())
- return;
- if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline)
- return;
-
- const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
- if (JT.empty()) return;
-
- unsigned NumEntries = 0;
- for (unsigned i = 0, e = JT.size(); i != e; ++i)
- NumEntries += JT[i].MBBs.size();
-
- unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getDataLayout());
-
- // Just allocate space for all the jump tables now. We will fix up the actual
- // MBB entries in the tables after we emit the code for each block, since then
- // we will know the final locations of the MBBs in memory.
- JumpTable = MJTI;
- JumpTableBase = allocateSpace(NumEntries * EntrySize,
- MJTI->getEntryAlignment(*TheJIT->getDataLayout()));
-}
-
-void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) {
- if (TheJIT->getJITInfo().hasCustomJumpTables())
- return;
-
- const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
- if (JT.empty() || !JumpTableBase) return;
-
-
- switch (MJTI->getEntryKind()) {
- case MachineJumpTableInfo::EK_Inline:
- return;
- case MachineJumpTableInfo::EK_BlockAddress: {
- // EK_BlockAddress - Each entry is a plain address of block, e.g.:
- // .word LBB123
- assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == sizeof(void*) &&
- "Cross JIT'ing?");
-
- // For each jump table, map each target in the jump table to the address of
- // an emitted MachineBasicBlock.
- intptr_t *SlotPtr = (intptr_t*)JumpTableBase;
-
- for (unsigned i = 0, e = JT.size(); i != e; ++i) {
- const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs;
- // Store the address of the basic block for this jump table slot in the
- // memory we allocated for the jump table in 'initJumpTableInfo'
- for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi)
- *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]);
- }
- break;
- }
-
- case MachineJumpTableInfo::EK_Custom32:
- case MachineJumpTableInfo::EK_GPRel32BlockAddress:
- case MachineJumpTableInfo::EK_LabelDifference32: {
- assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == 4&&"Cross JIT'ing?");
- // For each jump table, place the offset from the beginning of the table
- // to the target address.
- int *SlotPtr = (int*)JumpTableBase;
-
- for (unsigned i = 0, e = JT.size(); i != e; ++i) {
- const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs;
- // Store the offset of the basic block for this jump table slot in the
- // memory we allocated for the jump table in 'initJumpTableInfo'
- uintptr_t Base = (uintptr_t)SlotPtr;
- for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) {
- uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]);
- /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook.
- *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base);
- }
- }
- break;
- }
- case MachineJumpTableInfo::EK_GPRel64BlockAddress:
- llvm_unreachable(
- "JT Info emission not implemented for GPRel64BlockAddress yet.");
- }
-}
-
-void JITEmitter::startGVStub(const GlobalValue* GV,
- unsigned StubSize, unsigned Alignment) {
- SavedBufferBegin = BufferBegin;
- SavedBufferEnd = BufferEnd;
- SavedCurBufferPtr = CurBufferPtr;
-
- BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment);
- BufferEnd = BufferBegin+StubSize+1;
-}
-
-void JITEmitter::startGVStub(void *Buffer, unsigned StubSize) {
- SavedBufferBegin = BufferBegin;
- SavedBufferEnd = BufferEnd;
- SavedCurBufferPtr = CurBufferPtr;
-
- BufferBegin = CurBufferPtr = (uint8_t *)Buffer;
- BufferEnd = BufferBegin+StubSize+1;
-}
-
-void JITEmitter::finishGVStub() {
- assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space.");
- NumBytes += getCurrentPCOffset();
- BufferBegin = SavedBufferBegin;
- BufferEnd = SavedBufferEnd;
- CurBufferPtr = SavedCurBufferPtr;
-}
-
-void *JITEmitter::allocIndirectGV(const GlobalValue *GV,
- const uint8_t *Buffer, size_t Size,
- unsigned Alignment) {
- uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment);
- memcpy(IndGV, Buffer, Size);
- return IndGV;
-}
-
-// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry
-// in the constant pool that was last emitted with the 'emitConstantPool'
-// method.
-//
-uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const {
- assert(ConstantNum < ConstantPool->getConstants().size() &&
- "Invalid ConstantPoolIndex!");
- return ConstPoolAddresses[ConstantNum];
-}
-
-// getJumpTableEntryAddress - Return the address of the JumpTable with index
-// 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo'
-//
-uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const {
- const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables();
- assert(Index < JT.size() && "Invalid jump table index!");
-
- unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getDataLayout());
-
- unsigned Offset = 0;
- for (unsigned i = 0; i < Index; ++i)
- Offset += JT[i].MBBs.size();
-
- Offset *= EntrySize;
-
- return (uintptr_t)((char *)JumpTableBase + Offset);
-}
-
-void JITEmitter::EmittedFunctionConfig::onDelete(
- JITEmitter *Emitter, const Function *F) {
- Emitter->deallocateMemForFunction(F);
-}
-void JITEmitter::EmittedFunctionConfig::onRAUW(
- JITEmitter *, const Function*, const Function*) {
- llvm_unreachable("The JIT doesn't know how to handle a"
- " RAUW on a value it has emitted.");
-}
-
-
-//===----------------------------------------------------------------------===//
-// Public interface to this file
-//===----------------------------------------------------------------------===//
-
-JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM,
- TargetMachine &tm) {
- return new JITEmitter(jit, JMM, tm);
-}
-
-// getPointerToFunctionOrStub - If the specified function has been
-// code-gen'd, return a pointer to the function. If not, compile it, or use
-// a stub to implement lazy compilation if available.
-//
-void *JIT::getPointerToFunctionOrStub(Function *F) {
- // If we have already code generated the function, just return the address.
- if (void *Addr = getPointerToGlobalIfAvailable(F))
- return Addr;
-
- // Get a stub if the target supports it.
- JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter());
- return JE->getJITResolver().getLazyFunctionStub(F);
-}
-
-void JIT::updateFunctionStubUnlocked(Function *F) {
- // Get the empty stub we generated earlier.
- JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter());
- void *Stub = JE->getJITResolver().getLazyFunctionStub(F);
- void *Addr = getPointerToGlobalIfAvailable(F);
- assert(Addr != Stub && "Function must have non-stub address to be updated.");
-
- // Tell the target jit info to rewrite the stub at the specified address,
- // rather than creating a new one.
- TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout();
- JE->startGVStub(Stub, layout.Size);
- getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter());
- JE->finishGVStub();
-}
-
-/// freeMachineCodeForFunction - release machine code memory for given Function.
-///
-void JIT::freeMachineCodeForFunction(Function *F) {
- // Delete translation for this from the ExecutionEngine, so it will get
- // retranslated next time it is used.
- updateGlobalMapping(F, nullptr);
-
- // Free the actual memory for the function body and related stuff.
- static_cast<JITEmitter*>(JCE)->deallocateMemForFunction(F);
-}
+++ /dev/null
-//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the DefaultJITMemoryManager class.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ExecutionEngine/JITMemoryManager.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Config/config.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/Support/Allocator.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/DynamicLibrary.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/Memory.h"
-#include "llvm/Support/raw_ostream.h"
-#include <cassert>
-#include <climits>
-#include <cstring>
-#include <vector>
-
-#if defined(__linux__)
-#if defined(HAVE_SYS_STAT_H)
-#include <sys/stat.h>
-#endif
-#include <fcntl.h>
-#include <unistd.h>
-#endif
-
-using namespace llvm;
-
-#define DEBUG_TYPE "jit"
-
-STATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT");
-
-JITMemoryManager::~JITMemoryManager() {}
-
-//===----------------------------------------------------------------------===//
-// Memory Block Implementation.
-//===----------------------------------------------------------------------===//
-
-namespace {
- /// MemoryRangeHeader - For a range of memory, this is the header that we put
- /// on the block of memory. It is carefully crafted to be one word of memory.
- /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader
- /// which starts with this.
- struct FreeRangeHeader;
- struct MemoryRangeHeader {
- /// ThisAllocated - This is true if this block is currently allocated. If
- /// not, this can be converted to a FreeRangeHeader.
- unsigned ThisAllocated : 1;
-
- /// PrevAllocated - Keep track of whether the block immediately before us is
- /// allocated. If not, the word immediately before this header is the size
- /// of the previous block.
- unsigned PrevAllocated : 1;
-
- /// BlockSize - This is the size in bytes of this memory block,
- /// including this header.
- uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2);
-
-
- /// getBlockAfter - Return the memory block immediately after this one.
- ///
- MemoryRangeHeader &getBlockAfter() const {
- return *reinterpret_cast<MemoryRangeHeader *>(
- reinterpret_cast<char*>(
- const_cast<MemoryRangeHeader *>(this))+BlockSize);
- }
-
- /// getFreeBlockBefore - If the block before this one is free, return it,
- /// otherwise return null.
- FreeRangeHeader *getFreeBlockBefore() const {
- if (PrevAllocated) return nullptr;
- intptr_t PrevSize = reinterpret_cast<intptr_t *>(
- const_cast<MemoryRangeHeader *>(this))[-1];
- return reinterpret_cast<FreeRangeHeader *>(
- reinterpret_cast<char*>(
- const_cast<MemoryRangeHeader *>(this))-PrevSize);
- }
-
- /// FreeBlock - Turn an allocated block into a free block, adjusting
- /// bits in the object headers, and adding an end of region memory block.
- FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList);
-
- /// TrimAllocationToSize - If this allocated block is significantly larger
- /// than NewSize, split it into two pieces (where the former is NewSize
- /// bytes, including the header), and add the new block to the free list.
- FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList,
- uint64_t NewSize);
- };
-
- /// FreeRangeHeader - For a memory block that isn't already allocated, this
- /// keeps track of the current block and has a pointer to the next free block.
- /// Free blocks are kept on a circularly linked list.
- struct FreeRangeHeader : public MemoryRangeHeader {
- FreeRangeHeader *Prev;
- FreeRangeHeader *Next;
-
- /// getMinBlockSize - Get the minimum size for a memory block. Blocks
- /// smaller than this size cannot be created.
- static unsigned getMinBlockSize() {
- return sizeof(FreeRangeHeader)+sizeof(intptr_t);
- }
-
- /// SetEndOfBlockSizeMarker - The word at the end of every free block is
- /// known to be the size of the free block. Set it for this block.
- void SetEndOfBlockSizeMarker() {
- void *EndOfBlock = (char*)this + BlockSize;
- ((intptr_t *)EndOfBlock)[-1] = BlockSize;
- }
-
- FreeRangeHeader *RemoveFromFreeList() {
- assert(Next->Prev == this && Prev->Next == this && "Freelist broken!");
- Next->Prev = Prev;
- return Prev->Next = Next;
- }
-
- void AddToFreeList(FreeRangeHeader *FreeList) {
- Next = FreeList;
- Prev = FreeList->Prev;
- Prev->Next = this;
- Next->Prev = this;
- }
-
- /// GrowBlock - The block after this block just got deallocated. Merge it
- /// into the current block.
- void GrowBlock(uintptr_t NewSize);
-
- /// AllocateBlock - Mark this entire block allocated, updating freelists
- /// etc. This returns a pointer to the circular free-list.
- FreeRangeHeader *AllocateBlock();
- };
-}
-
-
-/// AllocateBlock - Mark this entire block allocated, updating freelists
-/// etc. This returns a pointer to the circular free-list.
-FreeRangeHeader *FreeRangeHeader::AllocateBlock() {
- assert(!ThisAllocated && !getBlockAfter().PrevAllocated &&
- "Cannot allocate an allocated block!");
- // Mark this block allocated.
- ThisAllocated = 1;
- getBlockAfter().PrevAllocated = 1;
-
- // Remove it from the free list.
- return RemoveFromFreeList();
-}
-
-/// FreeBlock - Turn an allocated block into a free block, adjusting
-/// bits in the object headers, and adding an end of region memory block.
-/// If possible, coalesce this block with neighboring blocks. Return the
-/// FreeRangeHeader to allocate from.
-FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) {
- MemoryRangeHeader *FollowingBlock = &getBlockAfter();
- assert(ThisAllocated && "This block is already free!");
- assert(FollowingBlock->PrevAllocated && "Flags out of sync!");
-
- FreeRangeHeader *FreeListToReturn = FreeList;
-
- // If the block after this one is free, merge it into this block.
- if (!FollowingBlock->ThisAllocated) {
- FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock;
- // "FreeList" always needs to be a valid free block. If we're about to
- // coalesce with it, update our notion of what the free list is.
- if (&FollowingFreeBlock == FreeList) {
- FreeList = FollowingFreeBlock.Next;
- FreeListToReturn = nullptr;
- assert(&FollowingFreeBlock != FreeList && "No tombstone block?");
- }
- FollowingFreeBlock.RemoveFromFreeList();
-
- // Include the following block into this one.
- BlockSize += FollowingFreeBlock.BlockSize;
- FollowingBlock = &FollowingFreeBlock.getBlockAfter();
-
- // Tell the block after the block we are coalescing that this block is
- // allocated.
- FollowingBlock->PrevAllocated = 1;
- }
-
- assert(FollowingBlock->ThisAllocated && "Missed coalescing?");
-
- if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) {
- PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize);
- return FreeListToReturn ? FreeListToReturn : PrevFreeBlock;
- }
-
- // Otherwise, mark this block free.
- FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this;
- FollowingBlock->PrevAllocated = 0;
- FreeBlock.ThisAllocated = 0;
-
- // Link this into the linked list of free blocks.
- FreeBlock.AddToFreeList(FreeList);
-
- // Add a marker at the end of the block, indicating the size of this free
- // block.
- FreeBlock.SetEndOfBlockSizeMarker();
- return FreeListToReturn ? FreeListToReturn : &FreeBlock;
-}
-
-/// GrowBlock - The block after this block just got deallocated. Merge it
-/// into the current block.
-void FreeRangeHeader::GrowBlock(uintptr_t NewSize) {
- assert(NewSize > BlockSize && "Not growing block?");
- BlockSize = NewSize;
- SetEndOfBlockSizeMarker();
- getBlockAfter().PrevAllocated = 0;
-}
-
-/// TrimAllocationToSize - If this allocated block is significantly larger
-/// than NewSize, split it into two pieces (where the former is NewSize
-/// bytes, including the header), and add the new block to the free list.
-FreeRangeHeader *MemoryRangeHeader::
-TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) {
- assert(ThisAllocated && getBlockAfter().PrevAllocated &&
- "Cannot deallocate part of an allocated block!");
-
- // Don't allow blocks to be trimmed below minimum required size
- NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize);
-
- // Round up size for alignment of header.
- unsigned HeaderAlign = __alignof(FreeRangeHeader);
- NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1);
-
- // Size is now the size of the block we will remove from the start of the
- // current block.
- assert(NewSize <= BlockSize &&
- "Allocating more space from this block than exists!");
-
- // If splitting this block will cause the remainder to be too small, do not
- // split the block.
- if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize())
- return FreeList;
-
- // Otherwise, we splice the required number of bytes out of this block, form
- // a new block immediately after it, then mark this block allocated.
- MemoryRangeHeader &FormerNextBlock = getBlockAfter();
-
- // Change the size of this block.
- BlockSize = NewSize;
-
- // Get the new block we just sliced out and turn it into a free block.
- FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter();
- NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock;
- NewNextBlock.ThisAllocated = 0;
- NewNextBlock.PrevAllocated = 1;
- NewNextBlock.SetEndOfBlockSizeMarker();
- FormerNextBlock.PrevAllocated = 0;
- NewNextBlock.AddToFreeList(FreeList);
- return &NewNextBlock;
-}
-
-//===----------------------------------------------------------------------===//
-// Memory Block Implementation.
-//===----------------------------------------------------------------------===//
-
-namespace {
-
- class DefaultJITMemoryManager;
-
- class JITAllocator {
- DefaultJITMemoryManager &JMM;
- public:
- JITAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { }
- void *Allocate(size_t Size, size_t /*Alignment*/);
- void Deallocate(void *Slab, size_t Size);
- };
-
- /// DefaultJITMemoryManager - Manage memory for the JIT code generation.
- /// This splits a large block of MAP_NORESERVE'd memory into two
- /// sections, one for function stubs, one for the functions themselves. We
- /// have to do this because we may need to emit a function stub while in the
- /// middle of emitting a function, and we don't know how large the function we
- /// are emitting is.
- class DefaultJITMemoryManager : public JITMemoryManager {
- public:
- /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at
- /// least this much unless more is requested. Currently, in 512k slabs.
- static const size_t DefaultCodeSlabSize = 512 * 1024;
-
- /// DefaultSlabSize - Allocate globals and stubs into slabs of 64K (probably
- /// 16 pages) unless we get an allocation above SizeThreshold.
- static const size_t DefaultSlabSize = 64 * 1024;
-
- /// DefaultSizeThreshold - For any allocation larger than 16K (probably
- /// 4 pages), we should allocate a separate slab to avoid wasted space at
- /// the end of a normal slab.
- static const size_t DefaultSizeThreshold = 16 * 1024;
-
- private:
- // Whether to poison freed memory.
- bool PoisonMemory;
-
- /// LastSlab - This points to the last slab allocated and is used as the
- /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all
- /// stubs, data, and code contiguously in memory. In general, however, this
- /// is not possible because the NearBlock parameter is ignored on Windows
- /// platforms and even on Unix it works on a best-effort pasis.
- sys::MemoryBlock LastSlab;
-
- // Memory slabs allocated by the JIT. We refer to them as slabs so we don't
- // confuse them with the blocks of memory described above.
- std::vector<sys::MemoryBlock> CodeSlabs;
- BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize,
- DefaultSizeThreshold> StubAllocator;
- BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize,
- DefaultSizeThreshold> DataAllocator;
-
- // Circular list of free blocks.
- FreeRangeHeader *FreeMemoryList;
-
- // When emitting code into a memory block, this is the block.
- MemoryRangeHeader *CurBlock;
-
- std::unique_ptr<uint8_t[]> GOTBase; // Target Specific reserved memory
- public:
- DefaultJITMemoryManager();
- ~DefaultJITMemoryManager();
-
- /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the
- /// last slab it allocated, so that subsequent allocations follow it.
- sys::MemoryBlock allocateNewSlab(size_t size);
-
- /// getPointerToNamedFunction - This method returns the address of the
- /// specified function by using the dlsym function call.
- void *getPointerToNamedFunction(const std::string &Name,
- bool AbortOnFailure = true) override;
-
- void AllocateGOT() override;
-
- // Testing methods.
- bool CheckInvariants(std::string &ErrorStr) override;
- size_t GetDefaultCodeSlabSize() override { return DefaultCodeSlabSize; }
- size_t GetDefaultDataSlabSize() override { return DefaultSlabSize; }
- size_t GetDefaultStubSlabSize() override { return DefaultSlabSize; }
- unsigned GetNumCodeSlabs() override { return CodeSlabs.size(); }
- unsigned GetNumDataSlabs() override { return DataAllocator.GetNumSlabs(); }
- unsigned GetNumStubSlabs() override { return StubAllocator.GetNumSlabs(); }
-
- /// startFunctionBody - When a function starts, allocate a block of free
- /// executable memory, returning a pointer to it and its actual size.
- uint8_t *startFunctionBody(const Function *F,
- uintptr_t &ActualSize) override {
-
- FreeRangeHeader* candidateBlock = FreeMemoryList;
- FreeRangeHeader* head = FreeMemoryList;
- FreeRangeHeader* iter = head->Next;
-
- uintptr_t largest = candidateBlock->BlockSize;
-
- // Search for the largest free block
- while (iter != head) {
- if (iter->BlockSize > largest) {
- largest = iter->BlockSize;
- candidateBlock = iter;
- }
- iter = iter->Next;
- }
-
- largest = largest - sizeof(MemoryRangeHeader);
-
- // If this block isn't big enough for the allocation desired, allocate
- // another block of memory and add it to the free list.
- if (largest < ActualSize ||
- largest <= FreeRangeHeader::getMinBlockSize()) {
- DEBUG(dbgs() << "JIT: Allocating another slab of memory for function.");
- candidateBlock = allocateNewCodeSlab((size_t)ActualSize);
- }
-
- // Select this candidate block for allocation
- CurBlock = candidateBlock;
-
- // Allocate the entire memory block.
- FreeMemoryList = candidateBlock->AllocateBlock();
- ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader);
- return (uint8_t *)(CurBlock + 1);
- }
-
- /// allocateNewCodeSlab - Helper method to allocate a new slab of code
- /// memory from the OS and add it to the free list. Returns the new
- /// FreeRangeHeader at the base of the slab.
- FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) {
- // If the user needs at least MinSize free memory, then we account for
- // two MemoryRangeHeaders: the one in the user's block, and the one at the
- // end of the slab.
- size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader);
- size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin);
- sys::MemoryBlock B = allocateNewSlab(SlabSize);
- CodeSlabs.push_back(B);
- char *MemBase = (char*)(B.base());
-
- // Put a tiny allocated block at the end of the memory chunk, so when
- // FreeBlock calls getBlockAfter it doesn't fall off the end.
- MemoryRangeHeader *EndBlock =
- (MemoryRangeHeader*)(MemBase + B.size()) - 1;
- EndBlock->ThisAllocated = 1;
- EndBlock->PrevAllocated = 0;
- EndBlock->BlockSize = sizeof(MemoryRangeHeader);
-
- // Start out with a vast new block of free memory.
- FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase;
- NewBlock->ThisAllocated = 0;
- // Make sure getFreeBlockBefore doesn't look into unmapped memory.
- NewBlock->PrevAllocated = 1;
- NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock;
- NewBlock->SetEndOfBlockSizeMarker();
- NewBlock->AddToFreeList(FreeMemoryList);
-
- assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize &&
- "The block was too small!");
- return NewBlock;
- }
-
- /// endFunctionBody - The function F is now allocated, and takes the memory
- /// in the range [FunctionStart,FunctionEnd).
- void endFunctionBody(const Function *F, uint8_t *FunctionStart,
- uint8_t *FunctionEnd) override {
- assert(FunctionEnd > FunctionStart);
- assert(FunctionStart == (uint8_t *)(CurBlock+1) &&
- "Mismatched function start/end!");
-
- uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock;
-
- // Release the memory at the end of this block that isn't needed.
- FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
- }
-
- /// allocateSpace - Allocate a memory block of the given size. This method
- /// cannot be called between calls to startFunctionBody and endFunctionBody.
- uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) override {
- CurBlock = FreeMemoryList;
- FreeMemoryList = FreeMemoryList->AllocateBlock();
-
- uint8_t *result = (uint8_t *)(CurBlock + 1);
-
- if (Alignment == 0) Alignment = 1;
- result = (uint8_t*)(((intptr_t)result+Alignment-1) &
- ~(intptr_t)(Alignment-1));
-
- uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock;
- FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
-
- return result;
- }
-
- /// allocateStub - Allocate memory for a function stub.
- uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize,
- unsigned Alignment) override {
- return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment);
- }
-
- /// allocateGlobal - Allocate memory for a global.
- uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) override {
- return (uint8_t*)DataAllocator.Allocate(Size, Alignment);
- }
-
- /// allocateCodeSection - Allocate memory for a code section.
- uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
- unsigned SectionID,
- StringRef SectionName) override {
- // Grow the required block size to account for the block header
- Size += sizeof(*CurBlock);
-
- // Alignment handling.
- if (!Alignment)
- Alignment = 16;
- Size += Alignment - 1;
-
- FreeRangeHeader* candidateBlock = FreeMemoryList;
- FreeRangeHeader* head = FreeMemoryList;
- FreeRangeHeader* iter = head->Next;
-
- uintptr_t largest = candidateBlock->BlockSize;
-
- // Search for the largest free block.
- while (iter != head) {
- if (iter->BlockSize > largest) {
- largest = iter->BlockSize;
- candidateBlock = iter;
- }
- iter = iter->Next;
- }
-
- largest = largest - sizeof(MemoryRangeHeader);
-
- // If this block isn't big enough for the allocation desired, allocate
- // another block of memory and add it to the free list.
- if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) {
- DEBUG(dbgs() << "JIT: Allocating another slab of memory for function.");
- candidateBlock = allocateNewCodeSlab((size_t)Size);
- }
-
- // Select this candidate block for allocation
- CurBlock = candidateBlock;
-
- // Allocate the entire memory block.
- FreeMemoryList = candidateBlock->AllocateBlock();
- // Release the memory at the end of this block that isn't needed.
- FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size);
- uintptr_t unalignedAddr = (uintptr_t)CurBlock + sizeof(*CurBlock);
- return (uint8_t*)RoundUpToAlignment((uint64_t)unalignedAddr, Alignment);
- }
-
- /// allocateDataSection - Allocate memory for a data section.
- uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
- unsigned SectionID, StringRef SectionName,
- bool IsReadOnly) override {
- return (uint8_t*)DataAllocator.Allocate(Size, Alignment);
- }
-
- bool finalizeMemory(std::string *ErrMsg) override {
- return false;
- }
-
- uint8_t *getGOTBase() const override {
- return GOTBase.get();
- }
-
- void deallocateBlock(void *Block) {
- // Find the block that is allocated for this function.
- MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1;
- assert(MemRange->ThisAllocated && "Block isn't allocated!");
-
- // Fill the buffer with garbage!
- if (PoisonMemory) {
- memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
- }
-
- // Free the memory.
- FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
- }
-
- /// deallocateFunctionBody - Deallocate all memory for the specified
- /// function body.
- void deallocateFunctionBody(void *Body) override {
- if (Body) deallocateBlock(Body);
- }
-
- /// setMemoryWritable - When code generation is in progress,
- /// the code pages may need permissions changed.
- void setMemoryWritable() override {
- for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
- sys::Memory::setWritable(CodeSlabs[i]);
- }
- /// setMemoryExecutable - When code generation is done and we're ready to
- /// start execution, the code pages may need permissions changed.
- void setMemoryExecutable() override {
- for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
- sys::Memory::setExecutable(CodeSlabs[i]);
- }
-
- /// setPoisonMemory - Controls whether we write garbage over freed memory.
- ///
- void setPoisonMemory(bool poison) override {
- PoisonMemory = poison;
- }
- };
-}
-
-void *JITAllocator::Allocate(size_t Size, size_t /*Alignment*/) {
- sys::MemoryBlock B = JMM.allocateNewSlab(Size);
- return B.base();
-}
-
-void JITAllocator::Deallocate(void *Slab, size_t Size) {
- sys::MemoryBlock B(Slab, Size);
- sys::Memory::ReleaseRWX(B);
-}
-
-DefaultJITMemoryManager::DefaultJITMemoryManager()
- :
-#ifdef NDEBUG
- PoisonMemory(false),
-#else
- PoisonMemory(true),
-#endif
- LastSlab(nullptr, 0), StubAllocator(*this), DataAllocator(*this) {
-
- // Allocate space for code.
- sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize);
- CodeSlabs.push_back(MemBlock);
- uint8_t *MemBase = (uint8_t*)MemBlock.base();
-
- // We set up the memory chunk with 4 mem regions, like this:
- // [ START
- // [ Free #0 ] -> Large space to allocate functions from.
- // [ Allocated #1 ] -> Tiny space to separate regions.
- // [ Free #2 ] -> Tiny space so there is always at least 1 free block.
- // [ Allocated #3 ] -> Tiny space to prevent looking past end of block.
- // END ]
- //
- // The last three blocks are never deallocated or touched.
-
- // Add MemoryRangeHeader to the end of the memory region, indicating that
- // the space after the block of memory is allocated. This is block #3.
- MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1;
- Mem3->ThisAllocated = 1;
- Mem3->PrevAllocated = 0;
- Mem3->BlockSize = sizeof(MemoryRangeHeader);
-
- /// Add a tiny free region so that the free list always has one entry.
- FreeRangeHeader *Mem2 =
- (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize());
- Mem2->ThisAllocated = 0;
- Mem2->PrevAllocated = 1;
- Mem2->BlockSize = FreeRangeHeader::getMinBlockSize();
- Mem2->SetEndOfBlockSizeMarker();
- Mem2->Prev = Mem2; // Mem2 *is* the free list for now.
- Mem2->Next = Mem2;
-
- /// Add a tiny allocated region so that Mem2 is never coalesced away.
- MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1;
- Mem1->ThisAllocated = 1;
- Mem1->PrevAllocated = 0;
- Mem1->BlockSize = sizeof(MemoryRangeHeader);
-
- // Add a FreeRangeHeader to the start of the function body region, indicating
- // that the space is free. Mark the previous block allocated so we never look
- // at it.
- FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase;
- Mem0->ThisAllocated = 0;
- Mem0->PrevAllocated = 1;
- Mem0->BlockSize = (char*)Mem1-(char*)Mem0;
- Mem0->SetEndOfBlockSizeMarker();
- Mem0->AddToFreeList(Mem2);
-
- // Start out with the freelist pointing to Mem0.
- FreeMemoryList = Mem0;
-}
-
-void DefaultJITMemoryManager::AllocateGOT() {
- assert(!GOTBase && "Cannot allocate the got multiple times");
- GOTBase = make_unique<uint8_t[]>(sizeof(void*) * 8192);
- HasGOT = true;
-}
-
-DefaultJITMemoryManager::~DefaultJITMemoryManager() {
- for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
- sys::Memory::ReleaseRWX(CodeSlabs[i]);
-}
-
-sys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) {
- // Allocate a new block close to the last one.
- std::string ErrMsg;
- sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : nullptr;
- sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg);
- if (!B.base()) {
- report_fatal_error("Allocation failed when allocating new memory in the"
- " JIT\n" + Twine(ErrMsg));
- }
- LastSlab = B;
- ++NumSlabs;
- // Initialize the slab to garbage when debugging.
- if (PoisonMemory) {
- memset(B.base(), 0xCD, B.size());
- }
- return B;
-}
-
-/// CheckInvariants - For testing only. Return "" if all internal invariants
-/// are preserved, and a helpful error message otherwise. For free and
-/// allocated blocks, make sure that adding BlockSize gives a valid block.
-/// For free blocks, make sure they're in the free list and that their end of
-/// block size marker is correct. This function should return an error before
-/// accessing bad memory. This function is defined here instead of in
-/// JITMemoryManagerTest.cpp so that we don't have to expose all of the
-/// implementation details of DefaultJITMemoryManager.
-bool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) {
- raw_string_ostream Err(ErrorStr);
-
- // Construct the set of FreeRangeHeader pointers so we can query it
- // efficiently.
- llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet;
- FreeRangeHeader* FreeHead = FreeMemoryList;
- FreeRangeHeader* FreeRange = FreeHead;
-
- do {
- // Check that the free range pointer is in the blocks we've allocated.
- bool Found = false;
- for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(),
- E = CodeSlabs.end(); I != E && !Found; ++I) {
- char *Start = (char*)I->base();
- char *End = Start + I->size();
- Found = (Start <= (char*)FreeRange && (char*)FreeRange < End);
- }
- if (!Found) {
- Err << "Corrupt free list; points to " << FreeRange;
- return false;
- }
-
- if (FreeRange->Next->Prev != FreeRange) {
- Err << "Next and Prev pointers do not match.";
- return false;
- }
-
- // Otherwise, add it to the set.
- FreeHdrSet.insert(FreeRange);
- FreeRange = FreeRange->Next;
- } while (FreeRange != FreeHead);
-
- // Go over each block, and look at each MemoryRangeHeader.
- for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(),
- E = CodeSlabs.end(); I != E; ++I) {
- char *Start = (char*)I->base();
- char *End = Start + I->size();
-
- // Check each memory range.
- for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = nullptr;
- Start <= (char*)Hdr && (char*)Hdr < End;
- Hdr = &Hdr->getBlockAfter()) {
- if (Hdr->ThisAllocated == 0) {
- // Check that this range is in the free list.
- if (!FreeHdrSet.count(Hdr)) {
- Err << "Found free header at " << Hdr << " that is not in free list.";
- return false;
- }
-
- // Now make sure the size marker at the end of the block is correct.
- uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1;
- if (!(Start <= (char*)Marker && (char*)Marker < End)) {
- Err << "Block size in header points out of current MemoryBlock.";
- return false;
- }
- if (Hdr->BlockSize != *Marker) {
- Err << "End of block size marker (" << *Marker << ") "
- << "and BlockSize (" << Hdr->BlockSize << ") don't match.";
- return false;
- }
- }
-
- if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) {
- Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != "
- << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")";
- return false;
- } else if (!LastHdr && !Hdr->PrevAllocated) {
- Err << "The first header should have PrevAllocated true.";
- return false;
- }
-
- // Remember the last header.
- LastHdr = Hdr;
- }
- }
-
- // All invariants are preserved.
- return true;
-}
-
-//===----------------------------------------------------------------------===//
-// getPointerToNamedFunction() implementation.
-//===----------------------------------------------------------------------===//
-
-// AtExitHandlers - List of functions to call when the program exits,
-// registered with the atexit() library function.
-static std::vector<void (*)()> AtExitHandlers;
-
-/// runAtExitHandlers - Run any functions registered by the program's
-/// calls to atexit(3), which we intercept and store in
-/// AtExitHandlers.
-///
-static void runAtExitHandlers() {
- while (!AtExitHandlers.empty()) {
- void (*Fn)() = AtExitHandlers.back();
- AtExitHandlers.pop_back();
- Fn();
- }
-}
-
-//===----------------------------------------------------------------------===//
-// Function stubs that are invoked instead of certain library calls
-//
-// Force the following functions to be linked in to anything that uses the
-// JIT. This is a hack designed to work around the all-too-clever Glibc
-// strategy of making these functions work differently when inlined vs. when
-// not inlined, and hiding their real definitions in a separate archive file
-// that the dynamic linker can't see. For more info, search for
-// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274.
-#if defined(__linux__) && defined(__GLIBC__)
-/* stat functions are redirecting to __xstat with a version number. On x86-64
- * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat'
- * available as an exported symbol, so we have to add it explicitly.
- */
-namespace {
-class StatSymbols {
-public:
- StatSymbols() {
- sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat);
- sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat);
- sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat);
- sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64);
- sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64);
- sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64);
- sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64);
- sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64);
- sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64);
- sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit);
- sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod);
- }
-};
-}
-static StatSymbols initStatSymbols;
-#endif // __linux__
-
-// jit_exit - Used to intercept the "exit" library call.
-static void jit_exit(int Status) {
- runAtExitHandlers(); // Run atexit handlers...
- exit(Status);
-}
-
-// jit_atexit - Used to intercept the "atexit" library call.
-static int jit_atexit(void (*Fn)()) {
- AtExitHandlers.push_back(Fn); // Take note of atexit handler...
- return 0; // Always successful
-}
-
-static int jit_noop() {
- return 0;
-}
-
-//===----------------------------------------------------------------------===//
-//
-/// getPointerToNamedFunction - This method returns the address of the specified
-/// function by using the dynamic loader interface. As such it is only useful
-/// for resolving library symbols, not code generated symbols.
-///
-void *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name,
- bool AbortOnFailure) {
- // Check to see if this is one of the functions we want to intercept. Note,
- // we cast to intptr_t here to silence a -pedantic warning that complains
- // about casting a function pointer to a normal pointer.
- if (Name == "exit") return (void*)(intptr_t)&jit_exit;
- if (Name == "atexit") return (void*)(intptr_t)&jit_atexit;
-
- // We should not invoke parent's ctors/dtors from generated main()!
- // On Mingw and Cygwin, the symbol __main is resolved to
- // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors
- // (and register wrong callee's dtors with atexit(3)).
- // We expect ExecutionEngine::runStaticConstructorsDestructors()
- // is called before ExecutionEngine::runFunctionAsMain() is called.
- if (Name == "__main") return (void*)(intptr_t)&jit_noop;
-
- const char *NameStr = Name.c_str();
- // If this is an asm specifier, skip the sentinal.
- if (NameStr[0] == 1) ++NameStr;
-
- // If it's an external function, look it up in the process image...
- void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr);
- if (Ptr) return Ptr;
-
- // If it wasn't found and if it starts with an underscore ('_') character,
- // try again without the underscore.
- if (NameStr[0] == '_') {
- Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1);
- if (Ptr) return Ptr;
- }
-
- // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These
- // are references to hidden visibility symbols that dlsym cannot resolve.
- // If we have one of these, strip off $LDBLStub and try again.
-#if defined(__APPLE__) && defined(__ppc__)
- if (Name.size() > 9 && Name[Name.size()-9] == '$' &&
- memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) {
- // First try turning $LDBLStub into $LDBL128. If that fails, strip it off.
- // This mirrors logic in libSystemStubs.a.
- std::string Prefix = std::string(Name.begin(), Name.end()-9);
- if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false))
- return Ptr;
- if (void *Ptr = getPointerToNamedFunction(Prefix, false))
- return Ptr;
- }
-#endif
-
- if (AbortOnFailure) {
- report_fatal_error("Program used external function '"+Name+
- "' which could not be resolved!");
- }
- return nullptr;
-}
-
-
-
-JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() {
- return new DefaultJITMemoryManager();
-}
-
-const size_t DefaultJITMemoryManager::DefaultCodeSlabSize;
-const size_t DefaultJITMemoryManager::DefaultSlabSize;
-const size_t DefaultJITMemoryManager::DefaultSizeThreshold;
+++ /dev/null
-;===- ./lib/ExecutionEngine/JIT/LLVMBuild.txt ------------------*- Conf -*--===;
-;
-; The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-; http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = JIT
-parent = ExecutionEngine
-required_libraries = CodeGen Core ExecutionEngine Support
+++ /dev/null
-##===- lib/ExecutionEngine/JIT/Makefile --------------------*- Makefile -*-===##
-#
-# The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
-#
-##===----------------------------------------------------------------------===##
-
-LEVEL = ../../..
-LIBRARYNAME = LLVMJIT
-
-# Get the $(ARCH) setting
-include $(LEVEL)/Makefile.config
-
-# Enable the X86 JIT if compiling on X86
-ifeq ($(ARCH), x86)
- ENABLE_X86_JIT = 1
-endif
-
-# This flag can also be used on the command line to force inclusion
-# of the X86 JIT on non-X86 hosts
-ifdef ENABLE_X86_JIT
- CPPFLAGS += -DENABLE_X86_JIT
-endif
-
-# Enable the Sparc JIT if compiling on Sparc
-ifeq ($(ARCH), Sparc)
- ENABLE_SPARC_JIT = 1
-endif
-
-# This flag can also be used on the command line to force inclusion
-# of the Sparc JIT on non-Sparc hosts
-ifdef ENABLE_SPARC_JIT
- CPPFLAGS += -DENABLE_SPARC_JIT
-endif
-
-include $(LEVEL)/Makefile.common
;===------------------------------------------------------------------------===;
[common]
-subdirectories = Interpreter JIT MCJIT RuntimeDyld IntelJITEvents OProfileJIT
+subdirectories = Interpreter MCJIT RuntimeDyld IntelJITEvents OProfileJIT
[component_0]
type = Library
add_llvm_library(LLVMMCJIT
+ JITMemoryManager.cpp
MCJIT.cpp
SectionMemoryManager.cpp
)
--- /dev/null
+//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the DefaultJITMemoryManager class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/JITMemoryManager.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Config/config.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Memory.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+#include <climits>
+#include <cstring>
+#include <vector>
+
+#if defined(__linux__)
+#if defined(HAVE_SYS_STAT_H)
+#include <sys/stat.h>
+#endif
+#include <fcntl.h>
+#include <unistd.h>
+#endif
+
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+STATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT");
+
+JITMemoryManager::~JITMemoryManager() {}
+
+//===----------------------------------------------------------------------===//
+// Memory Block Implementation.
+//===----------------------------------------------------------------------===//
+
+namespace {
+ /// MemoryRangeHeader - For a range of memory, this is the header that we put
+ /// on the block of memory. It is carefully crafted to be one word of memory.
+ /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader
+ /// which starts with this.
+ struct FreeRangeHeader;
+ struct MemoryRangeHeader {
+ /// ThisAllocated - This is true if this block is currently allocated. If
+ /// not, this can be converted to a FreeRangeHeader.
+ unsigned ThisAllocated : 1;
+
+ /// PrevAllocated - Keep track of whether the block immediately before us is
+ /// allocated. If not, the word immediately before this header is the size
+ /// of the previous block.
+ unsigned PrevAllocated : 1;
+
+ /// BlockSize - This is the size in bytes of this memory block,
+ /// including this header.
+ uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2);
+
+
+ /// getBlockAfter - Return the memory block immediately after this one.
+ ///
+ MemoryRangeHeader &getBlockAfter() const {
+ return *reinterpret_cast<MemoryRangeHeader *>(
+ reinterpret_cast<char*>(
+ const_cast<MemoryRangeHeader *>(this))+BlockSize);
+ }
+
+ /// getFreeBlockBefore - If the block before this one is free, return it,
+ /// otherwise return null.
+ FreeRangeHeader *getFreeBlockBefore() const {
+ if (PrevAllocated) return nullptr;
+ intptr_t PrevSize = reinterpret_cast<intptr_t *>(
+ const_cast<MemoryRangeHeader *>(this))[-1];
+ return reinterpret_cast<FreeRangeHeader *>(
+ reinterpret_cast<char*>(
+ const_cast<MemoryRangeHeader *>(this))-PrevSize);
+ }
+
+ /// FreeBlock - Turn an allocated block into a free block, adjusting
+ /// bits in the object headers, and adding an end of region memory block.
+ FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList);
+
+ /// TrimAllocationToSize - If this allocated block is significantly larger
+ /// than NewSize, split it into two pieces (where the former is NewSize
+ /// bytes, including the header), and add the new block to the free list.
+ FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList,
+ uint64_t NewSize);
+ };
+
+ /// FreeRangeHeader - For a memory block that isn't already allocated, this
+ /// keeps track of the current block and has a pointer to the next free block.
+ /// Free blocks are kept on a circularly linked list.
+ struct FreeRangeHeader : public MemoryRangeHeader {
+ FreeRangeHeader *Prev;
+ FreeRangeHeader *Next;
+
+ /// getMinBlockSize - Get the minimum size for a memory block. Blocks
+ /// smaller than this size cannot be created.
+ static unsigned getMinBlockSize() {
+ return sizeof(FreeRangeHeader)+sizeof(intptr_t);
+ }
+
+ /// SetEndOfBlockSizeMarker - The word at the end of every free block is
+ /// known to be the size of the free block. Set it for this block.
+ void SetEndOfBlockSizeMarker() {
+ void *EndOfBlock = (char*)this + BlockSize;
+ ((intptr_t *)EndOfBlock)[-1] = BlockSize;
+ }
+
+ FreeRangeHeader *RemoveFromFreeList() {
+ assert(Next->Prev == this && Prev->Next == this && "Freelist broken!");
+ Next->Prev = Prev;
+ return Prev->Next = Next;
+ }
+
+ void AddToFreeList(FreeRangeHeader *FreeList) {
+ Next = FreeList;
+ Prev = FreeList->Prev;
+ Prev->Next = this;
+ Next->Prev = this;
+ }
+
+ /// GrowBlock - The block after this block just got deallocated. Merge it
+ /// into the current block.
+ void GrowBlock(uintptr_t NewSize);
+
+ /// AllocateBlock - Mark this entire block allocated, updating freelists
+ /// etc. This returns a pointer to the circular free-list.
+ FreeRangeHeader *AllocateBlock();
+ };
+}
+
+
+/// AllocateBlock - Mark this entire block allocated, updating freelists
+/// etc. This returns a pointer to the circular free-list.
+FreeRangeHeader *FreeRangeHeader::AllocateBlock() {
+ assert(!ThisAllocated && !getBlockAfter().PrevAllocated &&
+ "Cannot allocate an allocated block!");
+ // Mark this block allocated.
+ ThisAllocated = 1;
+ getBlockAfter().PrevAllocated = 1;
+
+ // Remove it from the free list.
+ return RemoveFromFreeList();
+}
+
+/// FreeBlock - Turn an allocated block into a free block, adjusting
+/// bits in the object headers, and adding an end of region memory block.
+/// If possible, coalesce this block with neighboring blocks. Return the
+/// FreeRangeHeader to allocate from.
+FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) {
+ MemoryRangeHeader *FollowingBlock = &getBlockAfter();
+ assert(ThisAllocated && "This block is already free!");
+ assert(FollowingBlock->PrevAllocated && "Flags out of sync!");
+
+ FreeRangeHeader *FreeListToReturn = FreeList;
+
+ // If the block after this one is free, merge it into this block.
+ if (!FollowingBlock->ThisAllocated) {
+ FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock;
+ // "FreeList" always needs to be a valid free block. If we're about to
+ // coalesce with it, update our notion of what the free list is.
+ if (&FollowingFreeBlock == FreeList) {
+ FreeList = FollowingFreeBlock.Next;
+ FreeListToReturn = nullptr;
+ assert(&FollowingFreeBlock != FreeList && "No tombstone block?");
+ }
+ FollowingFreeBlock.RemoveFromFreeList();
+
+ // Include the following block into this one.
+ BlockSize += FollowingFreeBlock.BlockSize;
+ FollowingBlock = &FollowingFreeBlock.getBlockAfter();
+
+ // Tell the block after the block we are coalescing that this block is
+ // allocated.
+ FollowingBlock->PrevAllocated = 1;
+ }
+
+ assert(FollowingBlock->ThisAllocated && "Missed coalescing?");
+
+ if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) {
+ PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize);
+ return FreeListToReturn ? FreeListToReturn : PrevFreeBlock;
+ }
+
+ // Otherwise, mark this block free.
+ FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this;
+ FollowingBlock->PrevAllocated = 0;
+ FreeBlock.ThisAllocated = 0;
+
+ // Link this into the linked list of free blocks.
+ FreeBlock.AddToFreeList(FreeList);
+
+ // Add a marker at the end of the block, indicating the size of this free
+ // block.
+ FreeBlock.SetEndOfBlockSizeMarker();
+ return FreeListToReturn ? FreeListToReturn : &FreeBlock;
+}
+
+/// GrowBlock - The block after this block just got deallocated. Merge it
+/// into the current block.
+void FreeRangeHeader::GrowBlock(uintptr_t NewSize) {
+ assert(NewSize > BlockSize && "Not growing block?");
+ BlockSize = NewSize;
+ SetEndOfBlockSizeMarker();
+ getBlockAfter().PrevAllocated = 0;
+}
+
+/// TrimAllocationToSize - If this allocated block is significantly larger
+/// than NewSize, split it into two pieces (where the former is NewSize
+/// bytes, including the header), and add the new block to the free list.
+FreeRangeHeader *MemoryRangeHeader::
+TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) {
+ assert(ThisAllocated && getBlockAfter().PrevAllocated &&
+ "Cannot deallocate part of an allocated block!");
+
+ // Don't allow blocks to be trimmed below minimum required size
+ NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize);
+
+ // Round up size for alignment of header.
+ unsigned HeaderAlign = __alignof(FreeRangeHeader);
+ NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1);
+
+ // Size is now the size of the block we will remove from the start of the
+ // current block.
+ assert(NewSize <= BlockSize &&
+ "Allocating more space from this block than exists!");
+
+ // If splitting this block will cause the remainder to be too small, do not
+ // split the block.
+ if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize())
+ return FreeList;
+
+ // Otherwise, we splice the required number of bytes out of this block, form
+ // a new block immediately after it, then mark this block allocated.
+ MemoryRangeHeader &FormerNextBlock = getBlockAfter();
+
+ // Change the size of this block.
+ BlockSize = NewSize;
+
+ // Get the new block we just sliced out and turn it into a free block.
+ FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter();
+ NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock;
+ NewNextBlock.ThisAllocated = 0;
+ NewNextBlock.PrevAllocated = 1;
+ NewNextBlock.SetEndOfBlockSizeMarker();
+ FormerNextBlock.PrevAllocated = 0;
+ NewNextBlock.AddToFreeList(FreeList);
+ return &NewNextBlock;
+}
+
+//===----------------------------------------------------------------------===//
+// Memory Block Implementation.
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+ class DefaultJITMemoryManager;
+
+ class JITAllocator {
+ DefaultJITMemoryManager &JMM;
+ public:
+ JITAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { }
+ void *Allocate(size_t Size, size_t /*Alignment*/);
+ void Deallocate(void *Slab, size_t Size);
+ };
+
+ /// DefaultJITMemoryManager - Manage memory for the JIT code generation.
+ /// This splits a large block of MAP_NORESERVE'd memory into two
+ /// sections, one for function stubs, one for the functions themselves. We
+ /// have to do this because we may need to emit a function stub while in the
+ /// middle of emitting a function, and we don't know how large the function we
+ /// are emitting is.
+ class DefaultJITMemoryManager : public JITMemoryManager {
+ public:
+ /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at
+ /// least this much unless more is requested. Currently, in 512k slabs.
+ static const size_t DefaultCodeSlabSize = 512 * 1024;
+
+ /// DefaultSlabSize - Allocate globals and stubs into slabs of 64K (probably
+ /// 16 pages) unless we get an allocation above SizeThreshold.
+ static const size_t DefaultSlabSize = 64 * 1024;
+
+ /// DefaultSizeThreshold - For any allocation larger than 16K (probably
+ /// 4 pages), we should allocate a separate slab to avoid wasted space at
+ /// the end of a normal slab.
+ static const size_t DefaultSizeThreshold = 16 * 1024;
+
+ private:
+ // Whether to poison freed memory.
+ bool PoisonMemory;
+
+ /// LastSlab - This points to the last slab allocated and is used as the
+ /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all
+ /// stubs, data, and code contiguously in memory. In general, however, this
+ /// is not possible because the NearBlock parameter is ignored on Windows
+ /// platforms and even on Unix it works on a best-effort pasis.
+ sys::MemoryBlock LastSlab;
+
+ // Memory slabs allocated by the JIT. We refer to them as slabs so we don't
+ // confuse them with the blocks of memory described above.
+ std::vector<sys::MemoryBlock> CodeSlabs;
+ BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize,
+ DefaultSizeThreshold> StubAllocator;
+ BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize,
+ DefaultSizeThreshold> DataAllocator;
+
+ // Circular list of free blocks.
+ FreeRangeHeader *FreeMemoryList;
+
+ // When emitting code into a memory block, this is the block.
+ MemoryRangeHeader *CurBlock;
+
+ std::unique_ptr<uint8_t[]> GOTBase; // Target Specific reserved memory
+ public:
+ DefaultJITMemoryManager();
+ ~DefaultJITMemoryManager();
+
+ /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the
+ /// last slab it allocated, so that subsequent allocations follow it.
+ sys::MemoryBlock allocateNewSlab(size_t size);
+
+ /// getPointerToNamedFunction - This method returns the address of the
+ /// specified function by using the dlsym function call.
+ void *getPointerToNamedFunction(const std::string &Name,
+ bool AbortOnFailure = true) override;
+
+ void AllocateGOT() override;
+
+ // Testing methods.
+ bool CheckInvariants(std::string &ErrorStr) override;
+ size_t GetDefaultCodeSlabSize() override { return DefaultCodeSlabSize; }
+ size_t GetDefaultDataSlabSize() override { return DefaultSlabSize; }
+ size_t GetDefaultStubSlabSize() override { return DefaultSlabSize; }
+ unsigned GetNumCodeSlabs() override { return CodeSlabs.size(); }
+ unsigned GetNumDataSlabs() override { return DataAllocator.GetNumSlabs(); }
+ unsigned GetNumStubSlabs() override { return StubAllocator.GetNumSlabs(); }
+
+ /// startFunctionBody - When a function starts, allocate a block of free
+ /// executable memory, returning a pointer to it and its actual size.
+ uint8_t *startFunctionBody(const Function *F,
+ uintptr_t &ActualSize) override {
+
+ FreeRangeHeader* candidateBlock = FreeMemoryList;
+ FreeRangeHeader* head = FreeMemoryList;
+ FreeRangeHeader* iter = head->Next;
+
+ uintptr_t largest = candidateBlock->BlockSize;
+
+ // Search for the largest free block
+ while (iter != head) {
+ if (iter->BlockSize > largest) {
+ largest = iter->BlockSize;
+ candidateBlock = iter;
+ }
+ iter = iter->Next;
+ }
+
+ largest = largest - sizeof(MemoryRangeHeader);
+
+ // If this block isn't big enough for the allocation desired, allocate
+ // another block of memory and add it to the free list.
+ if (largest < ActualSize ||
+ largest <= FreeRangeHeader::getMinBlockSize()) {
+ DEBUG(dbgs() << "JIT: Allocating another slab of memory for function.");
+ candidateBlock = allocateNewCodeSlab((size_t)ActualSize);
+ }
+
+ // Select this candidate block for allocation
+ CurBlock = candidateBlock;
+
+ // Allocate the entire memory block.
+ FreeMemoryList = candidateBlock->AllocateBlock();
+ ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader);
+ return (uint8_t *)(CurBlock + 1);
+ }
+
+ /// allocateNewCodeSlab - Helper method to allocate a new slab of code
+ /// memory from the OS and add it to the free list. Returns the new
+ /// FreeRangeHeader at the base of the slab.
+ FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) {
+ // If the user needs at least MinSize free memory, then we account for
+ // two MemoryRangeHeaders: the one in the user's block, and the one at the
+ // end of the slab.
+ size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader);
+ size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin);
+ sys::MemoryBlock B = allocateNewSlab(SlabSize);
+ CodeSlabs.push_back(B);
+ char *MemBase = (char*)(B.base());
+
+ // Put a tiny allocated block at the end of the memory chunk, so when
+ // FreeBlock calls getBlockAfter it doesn't fall off the end.
+ MemoryRangeHeader *EndBlock =
+ (MemoryRangeHeader*)(MemBase + B.size()) - 1;
+ EndBlock->ThisAllocated = 1;
+ EndBlock->PrevAllocated = 0;
+ EndBlock->BlockSize = sizeof(MemoryRangeHeader);
+
+ // Start out with a vast new block of free memory.
+ FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase;
+ NewBlock->ThisAllocated = 0;
+ // Make sure getFreeBlockBefore doesn't look into unmapped memory.
+ NewBlock->PrevAllocated = 1;
+ NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock;
+ NewBlock->SetEndOfBlockSizeMarker();
+ NewBlock->AddToFreeList(FreeMemoryList);
+
+ assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize &&
+ "The block was too small!");
+ return NewBlock;
+ }
+
+ /// endFunctionBody - The function F is now allocated, and takes the memory
+ /// in the range [FunctionStart,FunctionEnd).
+ void endFunctionBody(const Function *F, uint8_t *FunctionStart,
+ uint8_t *FunctionEnd) override {
+ assert(FunctionEnd > FunctionStart);
+ assert(FunctionStart == (uint8_t *)(CurBlock+1) &&
+ "Mismatched function start/end!");
+
+ uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock;
+
+ // Release the memory at the end of this block that isn't needed.
+ FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
+ }
+
+ /// allocateSpace - Allocate a memory block of the given size. This method
+ /// cannot be called between calls to startFunctionBody and endFunctionBody.
+ uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) override {
+ CurBlock = FreeMemoryList;
+ FreeMemoryList = FreeMemoryList->AllocateBlock();
+
+ uint8_t *result = (uint8_t *)(CurBlock + 1);
+
+ if (Alignment == 0) Alignment = 1;
+ result = (uint8_t*)(((intptr_t)result+Alignment-1) &
+ ~(intptr_t)(Alignment-1));
+
+ uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock;
+ FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
+
+ return result;
+ }
+
+ /// allocateStub - Allocate memory for a function stub.
+ uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize,
+ unsigned Alignment) override {
+ return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment);
+ }
+
+ /// allocateGlobal - Allocate memory for a global.
+ uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) override {
+ return (uint8_t*)DataAllocator.Allocate(Size, Alignment);
+ }
+
+ /// allocateCodeSection - Allocate memory for a code section.
+ uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
+ unsigned SectionID,
+ StringRef SectionName) override {
+ // Grow the required block size to account for the block header
+ Size += sizeof(*CurBlock);
+
+ // Alignment handling.
+ if (!Alignment)
+ Alignment = 16;
+ Size += Alignment - 1;
+
+ FreeRangeHeader* candidateBlock = FreeMemoryList;
+ FreeRangeHeader* head = FreeMemoryList;
+ FreeRangeHeader* iter = head->Next;
+
+ uintptr_t largest = candidateBlock->BlockSize;
+
+ // Search for the largest free block.
+ while (iter != head) {
+ if (iter->BlockSize > largest) {
+ largest = iter->BlockSize;
+ candidateBlock = iter;
+ }
+ iter = iter->Next;
+ }
+
+ largest = largest - sizeof(MemoryRangeHeader);
+
+ // If this block isn't big enough for the allocation desired, allocate
+ // another block of memory and add it to the free list.
+ if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) {
+ DEBUG(dbgs() << "JIT: Allocating another slab of memory for function.");
+ candidateBlock = allocateNewCodeSlab((size_t)Size);
+ }
+
+ // Select this candidate block for allocation
+ CurBlock = candidateBlock;
+
+ // Allocate the entire memory block.
+ FreeMemoryList = candidateBlock->AllocateBlock();
+ // Release the memory at the end of this block that isn't needed.
+ FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size);
+ uintptr_t unalignedAddr = (uintptr_t)CurBlock + sizeof(*CurBlock);
+ return (uint8_t*)RoundUpToAlignment((uint64_t)unalignedAddr, Alignment);
+ }
+
+ /// allocateDataSection - Allocate memory for a data section.
+ uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
+ unsigned SectionID, StringRef SectionName,
+ bool IsReadOnly) override {
+ return (uint8_t*)DataAllocator.Allocate(Size, Alignment);
+ }
+
+ bool finalizeMemory(std::string *ErrMsg) override {
+ return false;
+ }
+
+ uint8_t *getGOTBase() const override {
+ return GOTBase.get();
+ }
+
+ void deallocateBlock(void *Block) {
+ // Find the block that is allocated for this function.
+ MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1;
+ assert(MemRange->ThisAllocated && "Block isn't allocated!");
+
+ // Fill the buffer with garbage!
+ if (PoisonMemory) {
+ memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
+ }
+
+ // Free the memory.
+ FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
+ }
+
+ /// deallocateFunctionBody - Deallocate all memory for the specified
+ /// function body.
+ void deallocateFunctionBody(void *Body) override {
+ if (Body) deallocateBlock(Body);
+ }
+
+ /// setMemoryWritable - When code generation is in progress,
+ /// the code pages may need permissions changed.
+ void setMemoryWritable() override {
+ for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
+ sys::Memory::setWritable(CodeSlabs[i]);
+ }
+ /// setMemoryExecutable - When code generation is done and we're ready to
+ /// start execution, the code pages may need permissions changed.
+ void setMemoryExecutable() override {
+ for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
+ sys::Memory::setExecutable(CodeSlabs[i]);
+ }
+
+ /// setPoisonMemory - Controls whether we write garbage over freed memory.
+ ///
+ void setPoisonMemory(bool poison) override {
+ PoisonMemory = poison;
+ }
+ };
+}
+
+void *JITAllocator::Allocate(size_t Size, size_t /*Alignment*/) {
+ sys::MemoryBlock B = JMM.allocateNewSlab(Size);
+ return B.base();
+}
+
+void JITAllocator::Deallocate(void *Slab, size_t Size) {
+ sys::MemoryBlock B(Slab, Size);
+ sys::Memory::ReleaseRWX(B);
+}
+
+DefaultJITMemoryManager::DefaultJITMemoryManager()
+ :
+#ifdef NDEBUG
+ PoisonMemory(false),
+#else
+ PoisonMemory(true),
+#endif
+ LastSlab(nullptr, 0), StubAllocator(*this), DataAllocator(*this) {
+
+ // Allocate space for code.
+ sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize);
+ CodeSlabs.push_back(MemBlock);
+ uint8_t *MemBase = (uint8_t*)MemBlock.base();
+
+ // We set up the memory chunk with 4 mem regions, like this:
+ // [ START
+ // [ Free #0 ] -> Large space to allocate functions from.
+ // [ Allocated #1 ] -> Tiny space to separate regions.
+ // [ Free #2 ] -> Tiny space so there is always at least 1 free block.
+ // [ Allocated #3 ] -> Tiny space to prevent looking past end of block.
+ // END ]
+ //
+ // The last three blocks are never deallocated or touched.
+
+ // Add MemoryRangeHeader to the end of the memory region, indicating that
+ // the space after the block of memory is allocated. This is block #3.
+ MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1;
+ Mem3->ThisAllocated = 1;
+ Mem3->PrevAllocated = 0;
+ Mem3->BlockSize = sizeof(MemoryRangeHeader);
+
+ /// Add a tiny free region so that the free list always has one entry.
+ FreeRangeHeader *Mem2 =
+ (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize());
+ Mem2->ThisAllocated = 0;
+ Mem2->PrevAllocated = 1;
+ Mem2->BlockSize = FreeRangeHeader::getMinBlockSize();
+ Mem2->SetEndOfBlockSizeMarker();
+ Mem2->Prev = Mem2; // Mem2 *is* the free list for now.
+ Mem2->Next = Mem2;
+
+ /// Add a tiny allocated region so that Mem2 is never coalesced away.
+ MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1;
+ Mem1->ThisAllocated = 1;
+ Mem1->PrevAllocated = 0;
+ Mem1->BlockSize = sizeof(MemoryRangeHeader);
+
+ // Add a FreeRangeHeader to the start of the function body region, indicating
+ // that the space is free. Mark the previous block allocated so we never look
+ // at it.
+ FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase;
+ Mem0->ThisAllocated = 0;
+ Mem0->PrevAllocated = 1;
+ Mem0->BlockSize = (char*)Mem1-(char*)Mem0;
+ Mem0->SetEndOfBlockSizeMarker();
+ Mem0->AddToFreeList(Mem2);
+
+ // Start out with the freelist pointing to Mem0.
+ FreeMemoryList = Mem0;
+}
+
+void DefaultJITMemoryManager::AllocateGOT() {
+ assert(!GOTBase && "Cannot allocate the got multiple times");
+ GOTBase = make_unique<uint8_t[]>(sizeof(void*) * 8192);
+ HasGOT = true;
+}
+
+DefaultJITMemoryManager::~DefaultJITMemoryManager() {
+ for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
+ sys::Memory::ReleaseRWX(CodeSlabs[i]);
+}
+
+sys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) {
+ // Allocate a new block close to the last one.
+ std::string ErrMsg;
+ sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : nullptr;
+ sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg);
+ if (!B.base()) {
+ report_fatal_error("Allocation failed when allocating new memory in the"
+ " JIT\n" + Twine(ErrMsg));
+ }
+ LastSlab = B;
+ ++NumSlabs;
+ // Initialize the slab to garbage when debugging.
+ if (PoisonMemory) {
+ memset(B.base(), 0xCD, B.size());
+ }
+ return B;
+}
+
+/// CheckInvariants - For testing only. Return "" if all internal invariants
+/// are preserved, and a helpful error message otherwise. For free and
+/// allocated blocks, make sure that adding BlockSize gives a valid block.
+/// For free blocks, make sure they're in the free list and that their end of
+/// block size marker is correct. This function should return an error before
+/// accessing bad memory. This function is defined here instead of in
+/// JITMemoryManagerTest.cpp so that we don't have to expose all of the
+/// implementation details of DefaultJITMemoryManager.
+bool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) {
+ raw_string_ostream Err(ErrorStr);
+
+ // Construct the set of FreeRangeHeader pointers so we can query it
+ // efficiently.
+ llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet;
+ FreeRangeHeader* FreeHead = FreeMemoryList;
+ FreeRangeHeader* FreeRange = FreeHead;
+
+ do {
+ // Check that the free range pointer is in the blocks we've allocated.
+ bool Found = false;
+ for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(),
+ E = CodeSlabs.end(); I != E && !Found; ++I) {
+ char *Start = (char*)I->base();
+ char *End = Start + I->size();
+ Found = (Start <= (char*)FreeRange && (char*)FreeRange < End);
+ }
+ if (!Found) {
+ Err << "Corrupt free list; points to " << FreeRange;
+ return false;
+ }
+
+ if (FreeRange->Next->Prev != FreeRange) {
+ Err << "Next and Prev pointers do not match.";
+ return false;
+ }
+
+ // Otherwise, add it to the set.
+ FreeHdrSet.insert(FreeRange);
+ FreeRange = FreeRange->Next;
+ } while (FreeRange != FreeHead);
+
+ // Go over each block, and look at each MemoryRangeHeader.
+ for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(),
+ E = CodeSlabs.end(); I != E; ++I) {
+ char *Start = (char*)I->base();
+ char *End = Start + I->size();
+
+ // Check each memory range.
+ for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = nullptr;
+ Start <= (char*)Hdr && (char*)Hdr < End;
+ Hdr = &Hdr->getBlockAfter()) {
+ if (Hdr->ThisAllocated == 0) {
+ // Check that this range is in the free list.
+ if (!FreeHdrSet.count(Hdr)) {
+ Err << "Found free header at " << Hdr << " that is not in free list.";
+ return false;
+ }
+
+ // Now make sure the size marker at the end of the block is correct.
+ uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1;
+ if (!(Start <= (char*)Marker && (char*)Marker < End)) {
+ Err << "Block size in header points out of current MemoryBlock.";
+ return false;
+ }
+ if (Hdr->BlockSize != *Marker) {
+ Err << "End of block size marker (" << *Marker << ") "
+ << "and BlockSize (" << Hdr->BlockSize << ") don't match.";
+ return false;
+ }
+ }
+
+ if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) {
+ Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != "
+ << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")";
+ return false;
+ } else if (!LastHdr && !Hdr->PrevAllocated) {
+ Err << "The first header should have PrevAllocated true.";
+ return false;
+ }
+
+ // Remember the last header.
+ LastHdr = Hdr;
+ }
+ }
+
+ // All invariants are preserved.
+ return true;
+}
+
+//===----------------------------------------------------------------------===//
+// getPointerToNamedFunction() implementation.
+//===----------------------------------------------------------------------===//
+
+// AtExitHandlers - List of functions to call when the program exits,
+// registered with the atexit() library function.
+static std::vector<void (*)()> AtExitHandlers;
+
+/// runAtExitHandlers - Run any functions registered by the program's
+/// calls to atexit(3), which we intercept and store in
+/// AtExitHandlers.
+///
+static void runAtExitHandlers() {
+ while (!AtExitHandlers.empty()) {
+ void (*Fn)() = AtExitHandlers.back();
+ AtExitHandlers.pop_back();
+ Fn();
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// Function stubs that are invoked instead of certain library calls
+//
+// Force the following functions to be linked in to anything that uses the
+// JIT. This is a hack designed to work around the all-too-clever Glibc
+// strategy of making these functions work differently when inlined vs. when
+// not inlined, and hiding their real definitions in a separate archive file
+// that the dynamic linker can't see. For more info, search for
+// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274.
+#if defined(__linux__) && defined(__GLIBC__)
+/* stat functions are redirecting to __xstat with a version number. On x86-64
+ * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat'
+ * available as an exported symbol, so we have to add it explicitly.
+ */
+namespace {
+class StatSymbols {
+public:
+ StatSymbols() {
+ sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat);
+ sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat);
+ sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat);
+ sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64);
+ sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64);
+ sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64);
+ sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64);
+ sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64);
+ sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64);
+ sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit);
+ sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod);
+ }
+};
+}
+static StatSymbols initStatSymbols;
+#endif // __linux__
+
+// jit_exit - Used to intercept the "exit" library call.
+static void jit_exit(int Status) {
+ runAtExitHandlers(); // Run atexit handlers...
+ exit(Status);
+}
+
+// jit_atexit - Used to intercept the "atexit" library call.
+static int jit_atexit(void (*Fn)()) {
+ AtExitHandlers.push_back(Fn); // Take note of atexit handler...
+ return 0; // Always successful
+}
+
+static int jit_noop() {
+ return 0;
+}
+
+//===----------------------------------------------------------------------===//
+//
+/// getPointerToNamedFunction - This method returns the address of the specified
+/// function by using the dynamic loader interface. As such it is only useful
+/// for resolving library symbols, not code generated symbols.
+///
+void *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name,
+ bool AbortOnFailure) {
+ // Check to see if this is one of the functions we want to intercept. Note,
+ // we cast to intptr_t here to silence a -pedantic warning that complains
+ // about casting a function pointer to a normal pointer.
+ if (Name == "exit") return (void*)(intptr_t)&jit_exit;
+ if (Name == "atexit") return (void*)(intptr_t)&jit_atexit;
+
+ // We should not invoke parent's ctors/dtors from generated main()!
+ // On Mingw and Cygwin, the symbol __main is resolved to
+ // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors
+ // (and register wrong callee's dtors with atexit(3)).
+ // We expect ExecutionEngine::runStaticConstructorsDestructors()
+ // is called before ExecutionEngine::runFunctionAsMain() is called.
+ if (Name == "__main") return (void*)(intptr_t)&jit_noop;
+
+ const char *NameStr = Name.c_str();
+ // If this is an asm specifier, skip the sentinal.
+ if (NameStr[0] == 1) ++NameStr;
+
+ // If it's an external function, look it up in the process image...
+ void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr);
+ if (Ptr) return Ptr;
+
+ // If it wasn't found and if it starts with an underscore ('_') character,
+ // try again without the underscore.
+ if (NameStr[0] == '_') {
+ Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1);
+ if (Ptr) return Ptr;
+ }
+
+ // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These
+ // are references to hidden visibility symbols that dlsym cannot resolve.
+ // If we have one of these, strip off $LDBLStub and try again.
+#if defined(__APPLE__) && defined(__ppc__)
+ if (Name.size() > 9 && Name[Name.size()-9] == '$' &&
+ memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) {
+ // First try turning $LDBLStub into $LDBL128. If that fails, strip it off.
+ // This mirrors logic in libSystemStubs.a.
+ std::string Prefix = std::string(Name.begin(), Name.end()-9);
+ if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false))
+ return Ptr;
+ if (void *Ptr = getPointerToNamedFunction(Prefix, false))
+ return Ptr;
+ }
+#endif
+
+ if (AbortOnFailure) {
+ report_fatal_error("Program used external function '"+Name+
+ "' which could not be resolved!");
+ }
+ return nullptr;
+}
+
+
+
+JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() {
+ return new DefaultJITMemoryManager();
+}
+
+const size_t DefaultJITMemoryManager::DefaultCodeSlabSize;
+const size_t DefaultJITMemoryManager::DefaultSlabSize;
+const size_t DefaultJITMemoryManager::DefaultSizeThreshold;
finalizeLoadedModules();
}
-void *MCJIT::getPointerToBasicBlock(BasicBlock *BB) {
- report_fatal_error("not yet implemented");
-}
-
uint64_t MCJIT::getExistingSymbolAddress(const std::string &Name) {
Mangler Mang(TM->getSubtargetImpl()->getDataLayout());
SmallString<128> FullName;
return (void*)Dyld.getSymbolLoadAddress(Name);
}
-void *MCJIT::recompileAndRelinkFunction(Function *F) {
- report_fatal_error("not yet implemented");
-}
-
-void MCJIT::freeMachineCodeForFunction(Function *F) {
- report_fatal_error("not yet implemented");
-}
-
void MCJIT::runStaticConstructorsDestructorsInModulePtrSet(
bool isDtors, ModulePtrSet::iterator I, ModulePtrSet::iterator E) {
for (; I != E; ++I) {
if (!L)
return;
MutexGuard locked(lock);
- SmallVector<JITEventListener*, 2>::reverse_iterator I=
- std::find(EventListeners.rbegin(), EventListeners.rend(), L);
+ auto I = std::find(EventListeners.rbegin(), EventListeners.rend(), L);
if (I != EventListeners.rend()) {
std::swap(*I, EventListeners.back());
EventListeners.pop_back();
void MCJIT::NotifyFreeingObject(const ObjectImage& Obj) {
MutexGuard locked(lock);
for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
- EventListeners[I]->NotifyFreeingObject(Obj);
+ JITEventListener *L = EventListeners[I];
+ L->NotifyFreeingObject(Obj);
}
}
MCContext *Ctx;
LinkingMemoryManager MemMgr;
RuntimeDyld Dyld;
- SmallVector<JITEventListener*, 2> EventListeners;
+ std::vector<JITEventListener*> EventListeners;
OwningModuleContainer OwnedModules;
/// \param isDtors - Run the destructors instead of constructors.
void runStaticConstructorsDestructors(bool isDtors) override;
- void *getPointerToBasicBlock(BasicBlock *BB) override;
-
void *getPointerToFunction(Function *F) override;
- void *recompileAndRelinkFunction(Function *F) override;
-
- void freeMachineCodeForFunction(Function *F) override;
-
GenericValue runFunction(Function *F,
const std::vector<GenericValue> &ArgValues) override;
include $(LEVEL)/Makefile.config
-PARALLEL_DIRS = Interpreter JIT MCJIT RuntimeDyld
+PARALLEL_DIRS = Interpreter MCJIT RuntimeDyld
ifeq ($(USE_INTEL_JITEVENTS), 1)
PARALLEL_DIRS += IntelJITEvents
// MCJIT can generate code for remote targets, but the old JIT and Interpreter
// must use the host architecture.
- if (UseMCJIT && WhichEngine != EngineKind::Interpreter && M)
+ if (WhichEngine != EngineKind::Interpreter && M)
TT.setTriple(M->getTargetTriple());
return selectTarget(TT, MArch, MCPU, MAttrs);
}
// FIXME: non-iOS ARM FastISel is broken with MCJIT.
- if (UseMCJIT &&
- TheTriple.getArch() == Triple::arm &&
+ if (TheTriple.getArch() == Triple::arm &&
!TheTriple.isiOS() &&
OptLevel == CodeGenOpt::None) {
OptLevel = CodeGenOpt::Less;
tablegen(LLVM AArch64GenRegisterInfo.inc -gen-register-info)
tablegen(LLVM AArch64GenInstrInfo.inc -gen-instr-info)
-tablegen(LLVM AArch64GenMCCodeEmitter.inc -gen-emitter -mc-emitter)
+tablegen(LLVM AArch64GenMCCodeEmitter.inc -gen-emitter)
tablegen(LLVM AArch64GenMCPseudoLowering.inc -gen-pseudo-lowering)
tablegen(LLVM AArch64GenAsmWriter.inc -gen-asm-writer)
tablegen(LLVM AArch64GenAsmWriter1.inc -gen-asm-writer -asmwriternum=1)
class ARMBaseTargetMachine;
class FunctionPass;
class ImmutablePass;
-class JITCodeEmitter;
class MachineInstr;
class MCInst;
class TargetLowering;
FunctionPass *createARMISelDag(ARMBaseTargetMachine &TM,
CodeGenOpt::Level OptLevel);
-
-FunctionPass *createARMJITCodeEmitterPass(ARMBaseTargetMachine &TM,
- JITCodeEmitter &JCE);
-
FunctionPass *createA15SDOptimizerPass();
FunctionPass *createARMLoadStoreOptimizationPass(bool PreAlloc = false);
FunctionPass *createARMExpandPseudoPass();
+++ /dev/null
-//===-- ARM/ARMCodeEmitter.cpp - Convert ARM code to machine code ---------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the pass that transforms the ARM machine instructions into
-// relocatable machine code.
-//
-//===----------------------------------------------------------------------===//
-
-#include "ARM.h"
-#include "ARMBaseInstrInfo.h"
-#include "ARMConstantPoolValue.h"
-#include "ARMMachineFunctionInfo.h"
-#include "ARMRelocations.h"
-#include "ARMSubtarget.h"
-#include "ARMTargetMachine.h"
-#include "MCTargetDesc/ARMAddressingModes.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/CodeGen/JITCodeEmitter.h"
-#include "llvm/CodeGen/MachineConstantPool.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineInstr.h"
-#include "llvm/CodeGen/MachineJumpTableInfo.h"
-#include "llvm/CodeGen/MachineModuleInfo.h"
-#include "llvm/CodeGen/Passes.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/Function.h"
-#include "llvm/PassManager.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/raw_ostream.h"
-#ifndef NDEBUG
-#include <iomanip>
-#endif
-using namespace llvm;
-
-#define DEBUG_TYPE "jit"
-
-STATISTIC(NumEmitted, "Number of machine instructions emitted");
-
-namespace {
-
- class ARMCodeEmitter : public MachineFunctionPass {
- ARMJITInfo *JTI;
- const ARMBaseInstrInfo *II;
- const DataLayout *TD;
- const ARMSubtarget *Subtarget;
- TargetMachine &TM;
- JITCodeEmitter &MCE;
- MachineModuleInfo *MMI;
- const std::vector<MachineConstantPoolEntry> *MCPEs;
- const std::vector<MachineJumpTableEntry> *MJTEs;
- bool IsPIC;
- bool IsThumb;
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<MachineModuleInfo>();
- MachineFunctionPass::getAnalysisUsage(AU);
- }
-
- static char ID;
- public:
- ARMCodeEmitter(TargetMachine &tm, JITCodeEmitter &mce)
- : MachineFunctionPass(ID), JTI(nullptr),
- II((const ARMBaseInstrInfo *)tm.getSubtargetImpl()->getInstrInfo()),
- TD(tm.getSubtargetImpl()->getDataLayout()), TM(tm), MCE(mce),
- MCPEs(nullptr), MJTEs(nullptr),
- IsPIC(TM.getRelocationModel() == Reloc::PIC_), IsThumb(false) {}
-
- /// getBinaryCodeForInstr - This function, generated by the
- /// CodeEmitterGenerator using TableGen, produces the binary encoding for
- /// machine instructions.
- uint64_t getBinaryCodeForInstr(const MachineInstr &MI) const;
-
- bool runOnMachineFunction(MachineFunction &MF) override;
-
- const char *getPassName() const override {
- return "ARM Machine Code Emitter";
- }
-
- void emitInstruction(const MachineInstr &MI);
-
- private:
-
- void emitWordLE(unsigned Binary);
- void emitDWordLE(uint64_t Binary);
- void emitConstPoolInstruction(const MachineInstr &MI);
- void emitMOVi32immInstruction(const MachineInstr &MI);
- void emitMOVi2piecesInstruction(const MachineInstr &MI);
- void emitLEApcrelJTInstruction(const MachineInstr &MI);
- void emitPseudoMoveInstruction(const MachineInstr &MI);
- void addPCLabel(unsigned LabelID);
- void emitPseudoInstruction(const MachineInstr &MI);
- unsigned getMachineSoRegOpValue(const MachineInstr &MI,
- const MCInstrDesc &MCID,
- const MachineOperand &MO,
- unsigned OpIdx);
-
- unsigned getMachineSoImmOpValue(unsigned SoImm);
- unsigned getAddrModeSBit(const MachineInstr &MI,
- const MCInstrDesc &MCID) const;
-
- void emitDataProcessingInstruction(const MachineInstr &MI,
- unsigned ImplicitRd = 0,
- unsigned ImplicitRn = 0);
-
- void emitLoadStoreInstruction(const MachineInstr &MI,
- unsigned ImplicitRd = 0,
- unsigned ImplicitRn = 0);
-
- void emitMiscLoadStoreInstruction(const MachineInstr &MI,
- unsigned ImplicitRn = 0);
-
- void emitLoadStoreMultipleInstruction(const MachineInstr &MI);
-
- void emitMulFrmInstruction(const MachineInstr &MI);
-
- void emitExtendInstruction(const MachineInstr &MI);
-
- void emitMiscArithInstruction(const MachineInstr &MI);
-
- void emitSaturateInstruction(const MachineInstr &MI);
-
- void emitBranchInstruction(const MachineInstr &MI);
-
- void emitInlineJumpTable(unsigned JTIndex);
-
- void emitMiscBranchInstruction(const MachineInstr &MI);
-
- void emitVFPArithInstruction(const MachineInstr &MI);
-
- void emitVFPConversionInstruction(const MachineInstr &MI);
-
- void emitVFPLoadStoreInstruction(const MachineInstr &MI);
-
- void emitVFPLoadStoreMultipleInstruction(const MachineInstr &MI);
-
- void emitNEONLaneInstruction(const MachineInstr &MI);
- void emitNEONDupInstruction(const MachineInstr &MI);
- void emitNEON1RegModImmInstruction(const MachineInstr &MI);
- void emitNEON2RegInstruction(const MachineInstr &MI);
- void emitNEON3RegInstruction(const MachineInstr &MI);
-
- /// getMachineOpValue - Return binary encoding of operand. If the machine
- /// operand requires relocation, record the relocation and return zero.
- unsigned getMachineOpValue(const MachineInstr &MI,
- const MachineOperand &MO) const;
- unsigned getMachineOpValue(const MachineInstr &MI, unsigned OpIdx) const {
- return getMachineOpValue(MI, MI.getOperand(OpIdx));
- }
-
- // FIXME: The legacy JIT ARMCodeEmitter doesn't rely on the the
- // TableGen'erated getBinaryCodeForInstr() function to encode any
- // operand values, instead querying getMachineOpValue() directly for
- // each operand it needs to encode. Thus, any of the new encoder
- // helper functions can simply return 0 as the values the return
- // are already handled elsewhere. They are placeholders to allow this
- // encoder to continue to function until the MC encoder is sufficiently
- // far along that this one can be eliminated entirely.
- unsigned NEONThumb2DataIPostEncoder(const MachineInstr &MI, unsigned Val)
- const { return 0; }
- unsigned NEONThumb2LoadStorePostEncoder(const MachineInstr &MI,unsigned Val)
- const { return 0; }
- unsigned NEONThumb2DupPostEncoder(const MachineInstr &MI,unsigned Val)
- const { return 0; }
- unsigned NEONThumb2V8PostEncoder(const MachineInstr &MI,unsigned Val)
- const { return 0; }
- unsigned VFPThumb2PostEncoder(const MachineInstr&MI, unsigned Val)
- const { return 0; }
- unsigned getAdrLabelOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getThumbAdrLabelOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getThumbBLTargetOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getThumbBLXTargetOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getThumbBRTargetOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getThumbBCCTargetOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getThumbCBTargetOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getBranchTargetOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getUnconditionalBranchTargetOpValue(const MachineInstr &MI,
- unsigned Op) const { return 0; }
- unsigned getARMBranchTargetOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getARMBLTargetOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getARMBLXTargetOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getCCOutOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getSOImmOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getT2SOImmOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getSORegRegOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getSORegImmOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getThumbAddrModeRegRegOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getT2AddrModeImm8OpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getT2Imm8s4OpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getT2AddrModeImm8s4OpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getT2AddrModeImm0_1020s4OpValue(const MachineInstr &MI,unsigned Op)
- const { return 0; }
- unsigned getT2AddrModeImm8OffsetOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getT2AddrModeSORegOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getT2SORegOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getT2AdrLabelOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getAddrMode6AddressOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getAddrMode6OneLane32AddressOpValue(const MachineInstr &MI,
- unsigned Op)
- const { return 0; }
- unsigned getAddrMode6DupAddressOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getAddrMode6OffsetOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getBitfieldInvertedMaskOpValue(const MachineInstr &MI,
- unsigned Op) const { return 0; }
- uint32_t getLdStSORegOpValue(const MachineInstr &MI, unsigned OpIdx)
- const { return 0; }
-
- unsigned getAddrModeImm12OpValue(const MachineInstr &MI, unsigned Op)
- const {
- // {17-13} = reg
- // {12} = (U)nsigned (add == '1', sub == '0')
- // {11-0} = imm12
- const MachineOperand &MO = MI.getOperand(Op);
- const MachineOperand &MO1 = MI.getOperand(Op + 1);
- if (!MO.isReg()) {
- emitConstPoolAddress(MO.getIndex(), ARM::reloc_arm_cp_entry);
- return 0;
- }
- unsigned Reg = II->getRegisterInfo().getEncodingValue(MO.getReg());
- int32_t Imm12 = MO1.getImm();
- uint32_t Binary;
- Binary = Imm12 & 0xfff;
- if (Imm12 >= 0)
- Binary |= (1 << 12);
- Binary |= (Reg << 13);
- return Binary;
- }
-
- unsigned getHiLo16ImmOpValue(const MachineInstr &MI, unsigned Op) const {
- return 0;
- }
-
- uint32_t getAddrMode2OffsetOpValue(const MachineInstr &MI, unsigned OpIdx)
- const { return 0;}
- uint32_t getPostIdxRegOpValue(const MachineInstr &MI, unsigned OpIdx)
- const { return 0;}
- uint32_t getAddrMode3OffsetOpValue(const MachineInstr &MI, unsigned OpIdx)
- const { return 0;}
- uint32_t getAddrMode3OpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- uint32_t getAddrModeThumbSPOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- uint32_t getAddrModeISOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- uint32_t getAddrModePCOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- uint32_t getAddrMode5OpValue(const MachineInstr &MI, unsigned Op) const {
- // {17-13} = reg
- // {12} = (U)nsigned (add == '1', sub == '0')
- // {11-0} = imm12
- const MachineOperand &MO = MI.getOperand(Op);
- const MachineOperand &MO1 = MI.getOperand(Op + 1);
- if (!MO.isReg()) {
- emitConstPoolAddress(MO.getIndex(), ARM::reloc_arm_cp_entry);
- return 0;
- }
- unsigned Reg = II->getRegisterInfo().getEncodingValue(MO.getReg());
- int32_t Imm12 = MO1.getImm();
-
- // Special value for #-0
- if (Imm12 == INT32_MIN)
- Imm12 = 0;
-
- // Immediate is always encoded as positive. The 'U' bit controls add vs
- // sub.
- bool isAdd = true;
- if (Imm12 < 0) {
- Imm12 = -Imm12;
- isAdd = false;
- }
-
- uint32_t Binary = Imm12 & 0xfff;
- if (isAdd)
- Binary |= (1 << 12);
- Binary |= (Reg << 13);
- return Binary;
- }
- unsigned getNEONVcvtImm32OpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
-
- unsigned getRegisterListOpValue(const MachineInstr &MI, unsigned Op)
- const { return 0; }
-
- unsigned getShiftRight8Imm(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getShiftRight16Imm(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getShiftRight32Imm(const MachineInstr &MI, unsigned Op)
- const { return 0; }
- unsigned getShiftRight64Imm(const MachineInstr &MI, unsigned Op)
- const { return 0; }
-
- /// getMovi32Value - Return binary encoding of operand for movw/movt. If the
- /// machine operand requires relocation, record the relocation and return
- /// zero.
- unsigned getMovi32Value(const MachineInstr &MI,const MachineOperand &MO,
- unsigned Reloc);
-
- /// getShiftOp - Return the shift opcode (bit[6:5]) of the immediate value.
- ///
- unsigned getShiftOp(unsigned Imm) const ;
-
- /// Routines that handle operands which add machine relocations which are
- /// fixed up by the relocation stage.
- void emitGlobalAddress(const GlobalValue *GV, unsigned Reloc,
- bool MayNeedFarStub, bool Indirect,
- intptr_t ACPV = 0) const;
- void emitExternalSymbolAddress(const char *ES, unsigned Reloc) const;
- void emitConstPoolAddress(unsigned CPI, unsigned Reloc) const;
- void emitJumpTableAddress(unsigned JTIndex, unsigned Reloc) const;
- void emitMachineBasicBlock(MachineBasicBlock *BB, unsigned Reloc,
- intptr_t JTBase = 0) const;
- unsigned encodeVFPRd(const MachineInstr &MI, unsigned OpIdx) const;
- unsigned encodeVFPRn(const MachineInstr &MI, unsigned OpIdx) const;
- unsigned encodeVFPRm(const MachineInstr &MI, unsigned OpIdx) const;
- unsigned encodeNEONRd(const MachineInstr &MI, unsigned OpIdx) const;
- unsigned encodeNEONRn(const MachineInstr &MI, unsigned OpIdx) const;
- unsigned encodeNEONRm(const MachineInstr &MI, unsigned OpIdx) const;
- };
-}
-
-char ARMCodeEmitter::ID = 0;
-
-/// createARMJITCodeEmitterPass - Return a pass that emits the collected ARM
-/// code to the specified MCE object.
-FunctionPass *llvm::createARMJITCodeEmitterPass(ARMBaseTargetMachine &TM,
- JITCodeEmitter &JCE) {
- return new ARMCodeEmitter(TM, JCE);
-}
-
-bool ARMCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
- TargetMachine &Target = const_cast<TargetMachine&>(MF.getTarget());
-
- assert((Target.getRelocationModel() != Reloc::Default ||
- Target.getRelocationModel() != Reloc::Static) &&
- "JIT relocation model must be set to static or default!");
- // Initialize the subtarget first so we can grab all of the
- // subtarget dependent variables from there.
- Subtarget = &TM.getSubtarget<ARMSubtarget>();
- JTI = static_cast<ARMJITInfo *>(Target.getSubtargetImpl()->getJITInfo());
- II = static_cast<const ARMBaseInstrInfo *>(Subtarget->getInstrInfo());
- TD = Target.getSubtargetImpl()->getDataLayout();
-
- MCPEs = &MF.getConstantPool()->getConstants();
- MJTEs = nullptr;
- if (MF.getJumpTableInfo()) MJTEs = &MF.getJumpTableInfo()->getJumpTables();
- IsPIC = TM.getRelocationModel() == Reloc::PIC_;
- IsThumb = MF.getInfo<ARMFunctionInfo>()->isThumbFunction();
- JTI->Initialize(MF, IsPIC);
- MMI = &getAnalysis<MachineModuleInfo>();
- MCE.setModuleInfo(MMI);
-
- do {
- DEBUG(errs() << "JITTing function '"
- << MF.getName() << "'\n");
- MCE.startFunction(MF);
- for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
- MBB != E; ++MBB) {
- MCE.StartMachineBasicBlock(MBB);
- for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
- I != E; ++I)
- emitInstruction(*I);
- }
- } while (MCE.finishFunction(MF));
-
- return false;
-}
-
-/// getShiftOp - Return the shift opcode (bit[6:5]) of the immediate value.
-///
-unsigned ARMCodeEmitter::getShiftOp(unsigned Imm) const {
- switch (ARM_AM::getAM2ShiftOpc(Imm)) {
- default: llvm_unreachable("Unknown shift opc!");
- case ARM_AM::asr: return 2;
- case ARM_AM::lsl: return 0;
- case ARM_AM::lsr: return 1;
- case ARM_AM::ror:
- case ARM_AM::rrx: return 3;
- }
-}
-
-/// getMovi32Value - Return binary encoding of operand for movw/movt. If the
-/// machine operand requires relocation, record the relocation and return zero.
-unsigned ARMCodeEmitter::getMovi32Value(const MachineInstr &MI,
- const MachineOperand &MO,
- unsigned Reloc) {
- assert(((Reloc == ARM::reloc_arm_movt) || (Reloc == ARM::reloc_arm_movw))
- && "Relocation to this function should be for movt or movw");
-
- if (MO.isImm())
- return static_cast<unsigned>(MO.getImm());
- else if (MO.isGlobal())
- emitGlobalAddress(MO.getGlobal(), Reloc, true, false);
- else if (MO.isSymbol())
- emitExternalSymbolAddress(MO.getSymbolName(), Reloc);
- else if (MO.isMBB())
- emitMachineBasicBlock(MO.getMBB(), Reloc);
- else {
-#ifndef NDEBUG
- errs() << MO;
-#endif
- llvm_unreachable("Unsupported operand type for movw/movt");
- }
- return 0;
-}
-
-/// getMachineOpValue - Return binary encoding of operand. If the machine
-/// operand requires relocation, record the relocation and return zero.
-unsigned ARMCodeEmitter::getMachineOpValue(const MachineInstr &MI,
- const MachineOperand &MO) const {
- if (MO.isReg())
- return II->getRegisterInfo().getEncodingValue(MO.getReg());
- else if (MO.isImm())
- return static_cast<unsigned>(MO.getImm());
- else if (MO.isGlobal())
- emitGlobalAddress(MO.getGlobal(), ARM::reloc_arm_branch, true, false);
- else if (MO.isSymbol())
- emitExternalSymbolAddress(MO.getSymbolName(), ARM::reloc_arm_branch);
- else if (MO.isCPI()) {
- const MCInstrDesc &MCID = MI.getDesc();
- // For VFP load, the immediate offset is multiplied by 4.
- unsigned Reloc = ((MCID.TSFlags & ARMII::FormMask) == ARMII::VFPLdStFrm)
- ? ARM::reloc_arm_vfp_cp_entry : ARM::reloc_arm_cp_entry;
- emitConstPoolAddress(MO.getIndex(), Reloc);
- } else if (MO.isJTI())
- emitJumpTableAddress(MO.getIndex(), ARM::reloc_arm_relative);
- else if (MO.isMBB())
- emitMachineBasicBlock(MO.getMBB(), ARM::reloc_arm_branch);
- else
- llvm_unreachable("Unable to encode MachineOperand!");
- return 0;
-}
-
-/// emitGlobalAddress - Emit the specified address to the code stream.
-///
-void ARMCodeEmitter::emitGlobalAddress(const GlobalValue *GV, unsigned Reloc,
- bool MayNeedFarStub, bool Indirect,
- intptr_t ACPV) const {
- MachineRelocation MR = Indirect
- ? MachineRelocation::getIndirectSymbol(MCE.getCurrentPCOffset(), Reloc,
- const_cast<GlobalValue *>(GV),
- ACPV, MayNeedFarStub)
- : MachineRelocation::getGV(MCE.getCurrentPCOffset(), Reloc,
- const_cast<GlobalValue *>(GV), ACPV,
- MayNeedFarStub);
- MCE.addRelocation(MR);
-}
-
-/// emitExternalSymbolAddress - Arrange for the address of an external symbol to
-/// be emitted to the current location in the function, and allow it to be PC
-/// relative.
-void ARMCodeEmitter::
-emitExternalSymbolAddress(const char *ES, unsigned Reloc) const {
- MCE.addRelocation(MachineRelocation::getExtSym(MCE.getCurrentPCOffset(),
- Reloc, ES));
-}
-
-/// emitConstPoolAddress - Arrange for the address of an constant pool
-/// to be emitted to the current location in the function, and allow it to be PC
-/// relative.
-void ARMCodeEmitter::emitConstPoolAddress(unsigned CPI, unsigned Reloc) const {
- // Tell JIT emitter we'll resolve the address.
- MCE.addRelocation(MachineRelocation::getConstPool(MCE.getCurrentPCOffset(),
- Reloc, CPI, 0, true));
-}
-
-/// emitJumpTableAddress - Arrange for the address of a jump table to
-/// be emitted to the current location in the function, and allow it to be PC
-/// relative.
-void ARMCodeEmitter::
-emitJumpTableAddress(unsigned JTIndex, unsigned Reloc) const {
- MCE.addRelocation(MachineRelocation::getJumpTable(MCE.getCurrentPCOffset(),
- Reloc, JTIndex, 0, true));
-}
-
-/// emitMachineBasicBlock - Emit the specified address basic block.
-void ARMCodeEmitter::emitMachineBasicBlock(MachineBasicBlock *BB,
- unsigned Reloc,
- intptr_t JTBase) const {
- MCE.addRelocation(MachineRelocation::getBB(MCE.getCurrentPCOffset(),
- Reloc, BB, JTBase));
-}
-
-void ARMCodeEmitter::emitWordLE(unsigned Binary) {
- DEBUG(errs() << " 0x";
- errs().write_hex(Binary) << "\n");
- MCE.emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitDWordLE(uint64_t Binary) {
- DEBUG(errs() << " 0x";
- errs().write_hex(Binary) << "\n");
- MCE.emitDWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitInstruction(const MachineInstr &MI) {
- DEBUG(errs() << "JIT: " << (void*)MCE.getCurrentPCValue() << ":\t" << MI);
-
- MCE.processDebugLoc(MI.getDebugLoc(), true);
-
- ++NumEmitted; // Keep track of the # of mi's emitted
- switch (MI.getDesc().TSFlags & ARMII::FormMask) {
- default: {
- llvm_unreachable("Unhandled instruction encoding format!");
- }
- case ARMII::MiscFrm:
- if (MI.getOpcode() == ARM::LEApcrelJT) {
- // Materialize jumptable address.
- emitLEApcrelJTInstruction(MI);
- break;
- }
- llvm_unreachable("Unhandled instruction encoding!");
- case ARMII::Pseudo:
- emitPseudoInstruction(MI);
- break;
- case ARMII::DPFrm:
- case ARMII::DPSoRegFrm:
- emitDataProcessingInstruction(MI);
- break;
- case ARMII::LdFrm:
- case ARMII::StFrm:
- emitLoadStoreInstruction(MI);
- break;
- case ARMII::LdMiscFrm:
- case ARMII::StMiscFrm:
- emitMiscLoadStoreInstruction(MI);
- break;
- case ARMII::LdStMulFrm:
- emitLoadStoreMultipleInstruction(MI);
- break;
- case ARMII::MulFrm:
- emitMulFrmInstruction(MI);
- break;
- case ARMII::ExtFrm:
- emitExtendInstruction(MI);
- break;
- case ARMII::ArithMiscFrm:
- emitMiscArithInstruction(MI);
- break;
- case ARMII::SatFrm:
- emitSaturateInstruction(MI);
- break;
- case ARMII::BrFrm:
- emitBranchInstruction(MI);
- break;
- case ARMII::BrMiscFrm:
- emitMiscBranchInstruction(MI);
- break;
- // VFP instructions.
- case ARMII::VFPUnaryFrm:
- case ARMII::VFPBinaryFrm:
- emitVFPArithInstruction(MI);
- break;
- case ARMII::VFPConv1Frm:
- case ARMII::VFPConv2Frm:
- case ARMII::VFPConv3Frm:
- case ARMII::VFPConv4Frm:
- case ARMII::VFPConv5Frm:
- emitVFPConversionInstruction(MI);
- break;
- case ARMII::VFPLdStFrm:
- emitVFPLoadStoreInstruction(MI);
- break;
- case ARMII::VFPLdStMulFrm:
- emitVFPLoadStoreMultipleInstruction(MI);
- break;
-
- // NEON instructions.
- case ARMII::NGetLnFrm:
- case ARMII::NSetLnFrm:
- emitNEONLaneInstruction(MI);
- break;
- case ARMII::NDupFrm:
- emitNEONDupInstruction(MI);
- break;
- case ARMII::N1RegModImmFrm:
- emitNEON1RegModImmInstruction(MI);
- break;
- case ARMII::N2RegFrm:
- emitNEON2RegInstruction(MI);
- break;
- case ARMII::N3RegFrm:
- emitNEON3RegInstruction(MI);
- break;
- }
- MCE.processDebugLoc(MI.getDebugLoc(), false);
-}
-
-void ARMCodeEmitter::emitConstPoolInstruction(const MachineInstr &MI) {
- unsigned CPI = MI.getOperand(0).getImm(); // CP instruction index.
- unsigned CPIndex = MI.getOperand(1).getIndex(); // Actual cp entry index.
- const MachineConstantPoolEntry &MCPE = (*MCPEs)[CPIndex];
-
- // Remember the CONSTPOOL_ENTRY address for later relocation.
- JTI->addConstantPoolEntryAddr(CPI, MCE.getCurrentPCValue());
-
- // Emit constpool island entry. In most cases, the actual values will be
- // resolved and relocated after code emission.
- if (MCPE.isMachineConstantPoolEntry()) {
- ARMConstantPoolValue *ACPV =
- static_cast<ARMConstantPoolValue*>(MCPE.Val.MachineCPVal);
-
- DEBUG(errs() << " ** ARM constant pool #" << CPI << " @ "
- << (void*)MCE.getCurrentPCValue() << " " << *ACPV << '\n');
-
- assert(ACPV->isGlobalValue() && "unsupported constant pool value");
- const GlobalValue *GV = cast<ARMConstantPoolConstant>(ACPV)->getGV();
- if (GV) {
- Reloc::Model RelocM = TM.getRelocationModel();
- emitGlobalAddress(GV, ARM::reloc_arm_machine_cp_entry,
- isa<Function>(GV),
- Subtarget->GVIsIndirectSymbol(GV, RelocM),
- (intptr_t)ACPV);
- } else {
- const char *Sym = cast<ARMConstantPoolSymbol>(ACPV)->getSymbol();
- emitExternalSymbolAddress(Sym, ARM::reloc_arm_absolute);
- }
- emitWordLE(0);
- } else {
- const Constant *CV = MCPE.Val.ConstVal;
-
- DEBUG({
- errs() << " ** Constant pool #" << CPI << " @ "
- << (void*)MCE.getCurrentPCValue() << " ";
- if (const Function *F = dyn_cast<Function>(CV))
- errs() << F->getName();
- else
- errs() << *CV;
- errs() << '\n';
- });
-
- if (const GlobalValue *GV = dyn_cast<GlobalValue>(CV)) {
- emitGlobalAddress(GV, ARM::reloc_arm_absolute, isa<Function>(GV), false);
- emitWordLE(0);
- } else if (const ConstantInt *CI = dyn_cast<ConstantInt>(CV)) {
- uint32_t Val = uint32_t(*CI->getValue().getRawData());
- emitWordLE(Val);
- } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CV)) {
- if (CFP->getType()->isFloatTy())
- emitWordLE(CFP->getValueAPF().bitcastToAPInt().getZExtValue());
- else if (CFP->getType()->isDoubleTy())
- emitDWordLE(CFP->getValueAPF().bitcastToAPInt().getZExtValue());
- else {
- llvm_unreachable("Unable to handle this constantpool entry!");
- }
- } else {
- llvm_unreachable("Unable to handle this constantpool entry!");
- }
- }
-}
-
-void ARMCodeEmitter::emitMOVi32immInstruction(const MachineInstr &MI) {
- const MachineOperand &MO0 = MI.getOperand(0);
- const MachineOperand &MO1 = MI.getOperand(1);
-
- // Emit the 'movw' instruction.
- unsigned Binary = 0x30 << 20; // mov: Insts{27-20} = 0b00110000
-
- unsigned Lo16 = getMovi32Value(MI, MO1, ARM::reloc_arm_movw) & 0xFFFF;
-
- // Set the conditional execution predicate.
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // Encode Rd.
- Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRdShift;
-
- // Encode imm16 as imm4:imm12
- Binary |= Lo16 & 0xFFF; // Insts{11-0} = imm12
- Binary |= ((Lo16 >> 12) & 0xF) << 16; // Insts{19-16} = imm4
- emitWordLE(Binary);
-
- unsigned Hi16 = getMovi32Value(MI, MO1, ARM::reloc_arm_movt) >> 16;
- // Emit the 'movt' instruction.
- Binary = 0x34 << 20; // movt: Insts{27-20} = 0b00110100
-
- // Set the conditional execution predicate.
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // Encode Rd.
- Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRdShift;
-
- // Encode imm16 as imm4:imm1, same as movw above.
- Binary |= Hi16 & 0xFFF;
- Binary |= ((Hi16 >> 12) & 0xF) << 16;
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitMOVi2piecesInstruction(const MachineInstr &MI) {
- const MachineOperand &MO0 = MI.getOperand(0);
- const MachineOperand &MO1 = MI.getOperand(1);
- assert(MO1.isImm() && ARM_AM::isSOImmTwoPartVal(MO1.getImm()) &&
- "Not a valid so_imm value!");
- unsigned V1 = ARM_AM::getSOImmTwoPartFirst(MO1.getImm());
- unsigned V2 = ARM_AM::getSOImmTwoPartSecond(MO1.getImm());
-
- // Emit the 'mov' instruction.
- unsigned Binary = 0xd << 21; // mov: Insts{24-21} = 0b1101
-
- // Set the conditional execution predicate.
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // Encode Rd.
- Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRdShift;
-
- // Encode so_imm.
- // Set bit I(25) to identify this is the immediate form of <shifter_op>
- Binary |= 1 << ARMII::I_BitShift;
- Binary |= getMachineSoImmOpValue(V1);
- emitWordLE(Binary);
-
- // Now the 'orr' instruction.
- Binary = 0xc << 21; // orr: Insts{24-21} = 0b1100
-
- // Set the conditional execution predicate.
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // Encode Rd.
- Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRdShift;
-
- // Encode Rn.
- Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRnShift;
-
- // Encode so_imm.
- // Set bit I(25) to identify this is the immediate form of <shifter_op>
- Binary |= 1 << ARMII::I_BitShift;
- Binary |= getMachineSoImmOpValue(V2);
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitLEApcrelJTInstruction(const MachineInstr &MI) {
- // It's basically add r, pc, (LJTI - $+8)
-
- const MCInstrDesc &MCID = MI.getDesc();
-
- // Emit the 'add' instruction.
- unsigned Binary = 0x4 << 21; // add: Insts{24-21} = 0b0100
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // Encode S bit if MI modifies CPSR.
- Binary |= getAddrModeSBit(MI, MCID);
-
- // Encode Rd.
- Binary |= getMachineOpValue(MI, 0) << ARMII::RegRdShift;
-
- // Encode Rn which is PC.
- Binary |= II->getRegisterInfo().getEncodingValue(ARM::PC) << ARMII::RegRnShift;
-
- // Encode the displacement.
- Binary |= 1 << ARMII::I_BitShift;
- emitJumpTableAddress(MI.getOperand(1).getIndex(), ARM::reloc_arm_jt_base);
-
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitPseudoMoveInstruction(const MachineInstr &MI) {
- unsigned Opcode = MI.getDesc().Opcode;
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // Encode S bit if MI modifies CPSR.
- if (Opcode == ARM::MOVsrl_flag || Opcode == ARM::MOVsra_flag)
- Binary |= 1 << ARMII::S_BitShift;
-
- // Encode register def if there is one.
- Binary |= getMachineOpValue(MI, 0) << ARMII::RegRdShift;
-
- // Encode the shift operation.
- switch (Opcode) {
- default: break;
- case ARM::RRX:
- // rrx
- Binary |= 0x6 << 4;
- break;
- case ARM::MOVsrl_flag:
- // lsr #1
- Binary |= (0x2 << 4) | (1 << 7);
- break;
- case ARM::MOVsra_flag:
- // asr #1
- Binary |= (0x4 << 4) | (1 << 7);
- break;
- }
-
- // Encode register Rm.
- Binary |= getMachineOpValue(MI, 1);
-
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::addPCLabel(unsigned LabelID) {
- DEBUG(errs() << " ** LPC" << LabelID << " @ "
- << (void*)MCE.getCurrentPCValue() << '\n');
- JTI->addPCLabelAddr(LabelID, MCE.getCurrentPCValue());
-}
-
-void ARMCodeEmitter::emitPseudoInstruction(const MachineInstr &MI) {
- unsigned Opcode = MI.getDesc().Opcode;
- switch (Opcode) {
- default:
- llvm_unreachable("ARMCodeEmitter::emitPseudoInstruction");
- case ARM::BX_CALL:
- case ARM::BMOVPCRX_CALL: {
- // First emit mov lr, pc
- unsigned Binary = 0x01a0e00f;
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
- emitWordLE(Binary);
-
- // and then emit the branch.
- emitMiscBranchInstruction(MI);
- break;
- }
- case TargetOpcode::INLINEASM: {
- // We allow inline assembler nodes with empty bodies - they can
- // implicitly define registers, which is ok for JIT.
- if (MI.getOperand(0).getSymbolName()[0]) {
- report_fatal_error("JIT does not support inline asm!");
- }
- break;
- }
- case TargetOpcode::CFI_INSTRUCTION:
- break;
- case TargetOpcode::EH_LABEL:
- MCE.emitLabel(MI.getOperand(0).getMCSymbol());
- break;
- case TargetOpcode::IMPLICIT_DEF:
- case TargetOpcode::KILL:
- // Do nothing.
- break;
- case ARM::CONSTPOOL_ENTRY:
- emitConstPoolInstruction(MI);
- break;
- case ARM::PICADD: {
- // Remember of the address of the PC label for relocation later.
- addPCLabel(MI.getOperand(2).getImm());
- // PICADD is just an add instruction that implicitly read pc.
- emitDataProcessingInstruction(MI, 0, ARM::PC);
- break;
- }
- case ARM::PICLDR:
- case ARM::PICLDRB:
- case ARM::PICSTR:
- case ARM::PICSTRB: {
- // Remember of the address of the PC label for relocation later.
- addPCLabel(MI.getOperand(2).getImm());
- // These are just load / store instructions that implicitly read pc.
- emitLoadStoreInstruction(MI, 0, ARM::PC);
- break;
- }
- case ARM::PICLDRH:
- case ARM::PICLDRSH:
- case ARM::PICLDRSB:
- case ARM::PICSTRH: {
- // Remember of the address of the PC label for relocation later.
- addPCLabel(MI.getOperand(2).getImm());
- // These are just load / store instructions that implicitly read pc.
- emitMiscLoadStoreInstruction(MI, ARM::PC);
- break;
- }
-
- case ARM::MOVi32imm:
- // Two instructions to materialize a constant.
- if (Subtarget->hasV6T2Ops())
- emitMOVi32immInstruction(MI);
- else
- emitMOVi2piecesInstruction(MI);
- break;
-
- case ARM::LEApcrelJT:
- // Materialize jumptable address.
- emitLEApcrelJTInstruction(MI);
- break;
- case ARM::RRX:
- case ARM::MOVsrl_flag:
- case ARM::MOVsra_flag:
- emitPseudoMoveInstruction(MI);
- break;
- }
-}
-
-unsigned ARMCodeEmitter::getMachineSoRegOpValue(const MachineInstr &MI,
- const MCInstrDesc &MCID,
- const MachineOperand &MO,
- unsigned OpIdx) {
- unsigned Binary = getMachineOpValue(MI, MO);
-
- const MachineOperand &MO1 = MI.getOperand(OpIdx + 1);
- const MachineOperand &MO2 = MI.getOperand(OpIdx + 2);
- ARM_AM::ShiftOpc SOpc = ARM_AM::getSORegShOp(MO2.getImm());
-
- // Encode the shift opcode.
- unsigned SBits = 0;
- unsigned Rs = MO1.getReg();
- if (Rs) {
- // Set shift operand (bit[7:4]).
- // LSL - 0001
- // LSR - 0011
- // ASR - 0101
- // ROR - 0111
- // RRX - 0110 and bit[11:8] clear.
- switch (SOpc) {
- default: llvm_unreachable("Unknown shift opc!");
- case ARM_AM::lsl: SBits = 0x1; break;
- case ARM_AM::lsr: SBits = 0x3; break;
- case ARM_AM::asr: SBits = 0x5; break;
- case ARM_AM::ror: SBits = 0x7; break;
- case ARM_AM::rrx: SBits = 0x6; break;
- }
- } else {
- // Set shift operand (bit[6:4]).
- // LSL - 000
- // LSR - 010
- // ASR - 100
- // ROR - 110
- switch (SOpc) {
- default: llvm_unreachable("Unknown shift opc!");
- case ARM_AM::lsl: SBits = 0x0; break;
- case ARM_AM::lsr: SBits = 0x2; break;
- case ARM_AM::asr: SBits = 0x4; break;
- case ARM_AM::ror: SBits = 0x6; break;
- }
- }
- Binary |= SBits << 4;
- if (SOpc == ARM_AM::rrx)
- return Binary;
-
- // Encode the shift operation Rs or shift_imm (except rrx).
- if (Rs) {
- // Encode Rs bit[11:8].
- assert(ARM_AM::getSORegOffset(MO2.getImm()) == 0);
- return Binary | (II->getRegisterInfo().getEncodingValue(Rs) << ARMII::RegRsShift);
- }
-
- // Encode shift_imm bit[11:7].
- return Binary | ARM_AM::getSORegOffset(MO2.getImm()) << 7;
-}
-
-unsigned ARMCodeEmitter::getMachineSoImmOpValue(unsigned SoImm) {
- int SoImmVal = ARM_AM::getSOImmVal(SoImm);
- assert(SoImmVal != -1 && "Not a valid so_imm value!");
-
- // Encode rotate_imm.
- unsigned Binary = (ARM_AM::getSOImmValRot((unsigned)SoImmVal) >> 1)
- << ARMII::SoRotImmShift;
-
- // Encode immed_8.
- Binary |= ARM_AM::getSOImmValImm((unsigned)SoImmVal);
- return Binary;
-}
-
-unsigned ARMCodeEmitter::getAddrModeSBit(const MachineInstr &MI,
- const MCInstrDesc &MCID) const {
- for (unsigned i = MI.getNumOperands(), e = MCID.getNumOperands(); i >= e;--i){
- const MachineOperand &MO = MI.getOperand(i-1);
- if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR)
- return 1 << ARMII::S_BitShift;
- }
- return 0;
-}
-
-void ARMCodeEmitter::emitDataProcessingInstruction(const MachineInstr &MI,
- unsigned ImplicitRd,
- unsigned ImplicitRn) {
- const MCInstrDesc &MCID = MI.getDesc();
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // Encode S bit if MI modifies CPSR.
- Binary |= getAddrModeSBit(MI, MCID);
-
- // Encode register def if there is one.
- unsigned NumDefs = MCID.getNumDefs();
- unsigned OpIdx = 0;
- if (NumDefs)
- Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
- else if (ImplicitRd)
- // Special handling for implicit use (e.g. PC).
- Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRd) << ARMII::RegRdShift);
-
- if (MCID.Opcode == ARM::MOVi16) {
- // Get immediate from MI.
- unsigned Lo16 = getMovi32Value(MI, MI.getOperand(OpIdx),
- ARM::reloc_arm_movw);
- // Encode imm which is the same as in emitMOVi32immInstruction().
- Binary |= Lo16 & 0xFFF;
- Binary |= ((Lo16 >> 12) & 0xF) << 16;
- emitWordLE(Binary);
- return;
- } else if(MCID.Opcode == ARM::MOVTi16) {
- unsigned Hi16 = (getMovi32Value(MI, MI.getOperand(OpIdx),
- ARM::reloc_arm_movt) >> 16);
- Binary |= Hi16 & 0xFFF;
- Binary |= ((Hi16 >> 12) & 0xF) << 16;
- emitWordLE(Binary);
- return;
- } else if ((MCID.Opcode == ARM::BFC) || (MCID.Opcode == ARM::BFI)) {
- uint32_t v = ~MI.getOperand(2).getImm();
- int32_t lsb = countTrailingZeros(v);
- int32_t msb = (32 - countLeadingZeros(v)) - 1;
- // Instr{20-16} = msb, Instr{11-7} = lsb
- Binary |= (msb & 0x1F) << 16;
- Binary |= (lsb & 0x1F) << 7;
- emitWordLE(Binary);
- return;
- } else if ((MCID.Opcode == ARM::UBFX) || (MCID.Opcode == ARM::SBFX)) {
- // Encode Rn in Instr{0-3}
- Binary |= getMachineOpValue(MI, OpIdx++);
-
- uint32_t lsb = MI.getOperand(OpIdx++).getImm();
- uint32_t widthm1 = MI.getOperand(OpIdx++).getImm() - 1;
-
- // Instr{20-16} = widthm1, Instr{11-7} = lsb
- Binary |= (widthm1 & 0x1F) << 16;
- Binary |= (lsb & 0x1F) << 7;
- emitWordLE(Binary);
- return;
- }
-
- // If this is a two-address operand, skip it. e.g. MOVCCr operand 1.
- if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
- ++OpIdx;
-
- // Encode first non-shifter register operand if there is one.
- bool isUnary = MCID.TSFlags & ARMII::UnaryDP;
- if (!isUnary) {
- if (ImplicitRn)
- // Special handling for implicit use (e.g. PC).
- Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRn) << ARMII::RegRnShift);
- else {
- Binary |= getMachineOpValue(MI, OpIdx) << ARMII::RegRnShift;
- ++OpIdx;
- }
- }
-
- // Encode shifter operand.
- const MachineOperand &MO = MI.getOperand(OpIdx);
- if ((MCID.TSFlags & ARMII::FormMask) == ARMII::DPSoRegFrm) {
- // Encode SoReg.
- emitWordLE(Binary | getMachineSoRegOpValue(MI, MCID, MO, OpIdx));
- return;
- }
-
- if (MO.isReg()) {
- // Encode register Rm.
- emitWordLE(Binary | II->getRegisterInfo().getEncodingValue(MO.getReg()));
- return;
- }
-
- // Encode so_imm.
- Binary |= getMachineSoImmOpValue((unsigned)MO.getImm());
-
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitLoadStoreInstruction(const MachineInstr &MI,
- unsigned ImplicitRd,
- unsigned ImplicitRn) {
- const MCInstrDesc &MCID = MI.getDesc();
- unsigned Form = MCID.TSFlags & ARMII::FormMask;
- bool IsPrePost = (MCID.TSFlags & ARMII::IndexModeMask) != 0;
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // If this is an LDRi12, STRi12 or LDRcp, nothing more needs be done.
- if (MI.getOpcode() == ARM::LDRi12 || MI.getOpcode() == ARM::LDRcp ||
- MI.getOpcode() == ARM::STRi12) {
- emitWordLE(Binary);
- return;
- }
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- unsigned OpIdx = 0;
-
- // Operand 0 of a pre- and post-indexed store is the address base
- // writeback. Skip it.
- bool Skipped = false;
- if (IsPrePost && Form == ARMII::StFrm) {
- ++OpIdx;
- Skipped = true;
- }
-
- // Set first operand
- if (ImplicitRd)
- // Special handling for implicit use (e.g. PC).
- Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRd) << ARMII::RegRdShift);
- else
- Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
-
- // Set second operand
- if (ImplicitRn)
- // Special handling for implicit use (e.g. PC).
- Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRn) << ARMII::RegRnShift);
- else
- Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
-
- // If this is a two-address operand, skip it. e.g. LDR_PRE.
- if (!Skipped && MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
- ++OpIdx;
-
- const MachineOperand &MO2 = MI.getOperand(OpIdx);
- unsigned AM2Opc = (ImplicitRn == ARM::PC)
- ? 0 : MI.getOperand(OpIdx+1).getImm();
-
- // Set bit U(23) according to sign of immed value (positive or negative).
- Binary |= ((ARM_AM::getAM2Op(AM2Opc) == ARM_AM::add ? 1 : 0) <<
- ARMII::U_BitShift);
- if (!MO2.getReg()) { // is immediate
- if (ARM_AM::getAM2Offset(AM2Opc))
- // Set the value of offset_12 field
- Binary |= ARM_AM::getAM2Offset(AM2Opc);
- emitWordLE(Binary);
- return;
- }
-
- // Set bit I(25), because this is not in immediate encoding.
- Binary |= 1 << ARMII::I_BitShift;
- assert(TargetRegisterInfo::isPhysicalRegister(MO2.getReg()));
- // Set bit[3:0] to the corresponding Rm register
- Binary |= II->getRegisterInfo().getEncodingValue(MO2.getReg());
-
- // If this instr is in scaled register offset/index instruction, set
- // shift_immed(bit[11:7]) and shift(bit[6:5]) fields.
- if (unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc)) {
- Binary |= getShiftOp(AM2Opc) << ARMII::ShiftImmShift; // shift
- Binary |= ShImm << ARMII::ShiftShift; // shift_immed
- }
-
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitMiscLoadStoreInstruction(const MachineInstr &MI,
- unsigned ImplicitRn) {
- const MCInstrDesc &MCID = MI.getDesc();
- unsigned Form = MCID.TSFlags & ARMII::FormMask;
- bool IsPrePost = (MCID.TSFlags & ARMII::IndexModeMask) != 0;
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- unsigned OpIdx = 0;
-
- // Operand 0 of a pre- and post-indexed store is the address base
- // writeback. Skip it.
- bool Skipped = false;
- if (IsPrePost && Form == ARMII::StMiscFrm) {
- ++OpIdx;
- Skipped = true;
- }
-
- // Set first operand
- Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
-
- // Skip LDRD and STRD's second operand.
- if (MCID.Opcode == ARM::LDRD || MCID.Opcode == ARM::STRD)
- ++OpIdx;
-
- // Set second operand
- if (ImplicitRn)
- // Special handling for implicit use (e.g. PC).
- Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRn) << ARMII::RegRnShift);
- else
- Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
-
- // If this is a two-address operand, skip it. e.g. LDRH_POST.
- if (!Skipped && MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
- ++OpIdx;
-
- const MachineOperand &MO2 = MI.getOperand(OpIdx);
- unsigned AM3Opc = (ImplicitRn == ARM::PC)
- ? 0 : MI.getOperand(OpIdx+1).getImm();
-
- // Set bit U(23) according to sign of immed value (positive or negative)
- Binary |= ((ARM_AM::getAM3Op(AM3Opc) == ARM_AM::add ? 1 : 0) <<
- ARMII::U_BitShift);
-
- // If this instr is in register offset/index encoding, set bit[3:0]
- // to the corresponding Rm register.
- if (MO2.getReg()) {
- Binary |= II->getRegisterInfo().getEncodingValue(MO2.getReg());
- emitWordLE(Binary);
- return;
- }
-
- // This instr is in immediate offset/index encoding, set bit 22 to 1.
- Binary |= 1 << ARMII::AM3_I_BitShift;
- if (unsigned ImmOffs = ARM_AM::getAM3Offset(AM3Opc)) {
- // Set operands
- Binary |= (ImmOffs >> 4) << ARMII::ImmHiShift; // immedH
- Binary |= (ImmOffs & 0xF); // immedL
- }
-
- emitWordLE(Binary);
-}
-
-static unsigned getAddrModeUPBits(unsigned Mode) {
- unsigned Binary = 0;
-
- // Set addressing mode by modifying bits U(23) and P(24)
- // IA - Increment after - bit U = 1 and bit P = 0
- // IB - Increment before - bit U = 1 and bit P = 1
- // DA - Decrement after - bit U = 0 and bit P = 0
- // DB - Decrement before - bit U = 0 and bit P = 1
- switch (Mode) {
- default: llvm_unreachable("Unknown addressing sub-mode!");
- case ARM_AM::da: break;
- case ARM_AM::db: Binary |= 0x1 << ARMII::P_BitShift; break;
- case ARM_AM::ia: Binary |= 0x1 << ARMII::U_BitShift; break;
- case ARM_AM::ib: Binary |= 0x3 << ARMII::U_BitShift; break;
- }
-
- return Binary;
-}
-
-void ARMCodeEmitter::emitLoadStoreMultipleInstruction(const MachineInstr &MI) {
- const MCInstrDesc &MCID = MI.getDesc();
- bool IsUpdating = (MCID.TSFlags & ARMII::IndexModeMask) != 0;
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // Skip operand 0 of an instruction with base register update.
- unsigned OpIdx = 0;
- if (IsUpdating)
- ++OpIdx;
-
- // Set base address operand
- Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
-
- // Set addressing mode by modifying bits U(23) and P(24)
- ARM_AM::AMSubMode Mode = ARM_AM::getLoadStoreMultipleSubMode(MI.getOpcode());
- Binary |= getAddrModeUPBits(ARM_AM::getAM4SubMode(Mode));
-
- // Set bit W(21)
- if (IsUpdating)
- Binary |= 0x1 << ARMII::W_BitShift;
-
- // Set registers
- for (unsigned i = OpIdx+2, e = MI.getNumOperands(); i != e; ++i) {
- const MachineOperand &MO = MI.getOperand(i);
- if (!MO.isReg() || MO.isImplicit())
- break;
- unsigned RegNum = II->getRegisterInfo().getEncodingValue(MO.getReg());
- assert(TargetRegisterInfo::isPhysicalRegister(MO.getReg()) &&
- RegNum < 16);
- Binary |= 0x1 << RegNum;
- }
-
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitMulFrmInstruction(const MachineInstr &MI) {
- const MCInstrDesc &MCID = MI.getDesc();
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // Encode S bit if MI modifies CPSR.
- Binary |= getAddrModeSBit(MI, MCID);
-
- // 32x32->64bit operations have two destination registers. The number
- // of register definitions will tell us if that's what we're dealing with.
- unsigned OpIdx = 0;
- if (MCID.getNumDefs() == 2)
- Binary |= getMachineOpValue (MI, OpIdx++) << ARMII::RegRdLoShift;
-
- // Encode Rd
- Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdHiShift;
-
- // Encode Rm
- Binary |= getMachineOpValue(MI, OpIdx++);
-
- // Encode Rs
- Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRsShift;
-
- // Many multiple instructions (e.g. MLA) have three src operands. Encode
- // it as Rn (for multiply, that's in the same offset as RdLo.
- if (MCID.getNumOperands() > OpIdx &&
- !MCID.OpInfo[OpIdx].isPredicate() &&
- !MCID.OpInfo[OpIdx].isOptionalDef())
- Binary |= getMachineOpValue(MI, OpIdx) << ARMII::RegRdLoShift;
-
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitExtendInstruction(const MachineInstr &MI) {
- const MCInstrDesc &MCID = MI.getDesc();
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- unsigned OpIdx = 0;
-
- // Encode Rd
- Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
-
- const MachineOperand &MO1 = MI.getOperand(OpIdx++);
- const MachineOperand &MO2 = MI.getOperand(OpIdx);
- if (MO2.isReg()) {
- // Two register operand form.
- // Encode Rn.
- Binary |= getMachineOpValue(MI, MO1) << ARMII::RegRnShift;
-
- // Encode Rm.
- Binary |= getMachineOpValue(MI, MO2);
- ++OpIdx;
- } else {
- Binary |= getMachineOpValue(MI, MO1);
- }
-
- // Encode rot imm (0, 8, 16, or 24) if it has a rotate immediate operand.
- if (MI.getOperand(OpIdx).isImm() &&
- !MCID.OpInfo[OpIdx].isPredicate() &&
- !MCID.OpInfo[OpIdx].isOptionalDef())
- Binary |= (getMachineOpValue(MI, OpIdx) / 8) << ARMII::ExtRotImmShift;
-
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitMiscArithInstruction(const MachineInstr &MI) {
- const MCInstrDesc &MCID = MI.getDesc();
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // PKH instructions are finished at this point
- if (MCID.Opcode == ARM::PKHBT || MCID.Opcode == ARM::PKHTB) {
- emitWordLE(Binary);
- return;
- }
-
- unsigned OpIdx = 0;
-
- // Encode Rd
- Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
-
- const MachineOperand &MO = MI.getOperand(OpIdx++);
- if (OpIdx == MCID.getNumOperands() ||
- MCID.OpInfo[OpIdx].isPredicate() ||
- MCID.OpInfo[OpIdx].isOptionalDef()) {
- // Encode Rm and it's done.
- Binary |= getMachineOpValue(MI, MO);
- emitWordLE(Binary);
- return;
- }
-
- // Encode Rn.
- Binary |= getMachineOpValue(MI, MO) << ARMII::RegRnShift;
-
- // Encode Rm.
- Binary |= getMachineOpValue(MI, OpIdx++);
-
- // Encode shift_imm.
- unsigned ShiftAmt = MI.getOperand(OpIdx).getImm();
- if (MCID.Opcode == ARM::PKHTB) {
- assert(ShiftAmt != 0 && "PKHTB shift_imm is 0!");
- if (ShiftAmt == 32)
- ShiftAmt = 0;
- }
- assert(ShiftAmt < 32 && "shift_imm range is 0 to 31!");
- Binary |= ShiftAmt << ARMII::ShiftShift;
-
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitSaturateInstruction(const MachineInstr &MI) {
- const MCInstrDesc &MCID = MI.getDesc();
-
- // Part of binary is determined by TableGen.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // Encode Rd
- Binary |= getMachineOpValue(MI, 0) << ARMII::RegRdShift;
-
- // Encode saturate bit position.
- unsigned Pos = MI.getOperand(1).getImm();
- if (MCID.Opcode == ARM::SSAT || MCID.Opcode == ARM::SSAT16)
- Pos -= 1;
- assert((Pos < 16 || (Pos < 32 &&
- MCID.Opcode != ARM::SSAT16 &&
- MCID.Opcode != ARM::USAT16)) &&
- "saturate bit position out of range");
- Binary |= Pos << 16;
-
- // Encode Rm
- Binary |= getMachineOpValue(MI, 2);
-
- // Encode shift_imm.
- if (MCID.getNumOperands() == 4) {
- unsigned ShiftOp = MI.getOperand(3).getImm();
- ARM_AM::ShiftOpc Opc = ARM_AM::getSORegShOp(ShiftOp);
- if (Opc == ARM_AM::asr)
- Binary |= (1 << 6);
- unsigned ShiftAmt = MI.getOperand(3).getImm();
- if (ShiftAmt == 32 && Opc == ARM_AM::asr)
- ShiftAmt = 0;
- assert(ShiftAmt < 32 && "shift_imm range is 0 to 31!");
- Binary |= ShiftAmt << ARMII::ShiftShift;
- }
-
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitBranchInstruction(const MachineInstr &MI) {
- const MCInstrDesc &MCID = MI.getDesc();
-
- if (MCID.Opcode == ARM::TPsoft) {
- llvm_unreachable("ARM::TPsoft FIXME"); // FIXME
- }
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // Set signed_immed_24 field
- Binary |= getMachineOpValue(MI, 0);
-
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitInlineJumpTable(unsigned JTIndex) {
- // Remember the base address of the inline jump table.
- uintptr_t JTBase = MCE.getCurrentPCValue();
- JTI->addJumpTableBaseAddr(JTIndex, JTBase);
- DEBUG(errs() << " ** Jump Table #" << JTIndex << " @ " << (void*)JTBase
- << '\n');
-
- // Now emit the jump table entries.
- const std::vector<MachineBasicBlock*> &MBBs = (*MJTEs)[JTIndex].MBBs;
- for (unsigned i = 0, e = MBBs.size(); i != e; ++i) {
- if (IsPIC)
- // DestBB address - JT base.
- emitMachineBasicBlock(MBBs[i], ARM::reloc_arm_pic_jt, JTBase);
- else
- // Absolute DestBB address.
- emitMachineBasicBlock(MBBs[i], ARM::reloc_arm_absolute);
- emitWordLE(0);
- }
-}
-
-void ARMCodeEmitter::emitMiscBranchInstruction(const MachineInstr &MI) {
- const MCInstrDesc &MCID = MI.getDesc();
-
- // Handle jump tables.
- if (MCID.Opcode == ARM::BR_JTr || MCID.Opcode == ARM::BR_JTadd) {
- // First emit a ldr pc, [] instruction.
- emitDataProcessingInstruction(MI, ARM::PC);
-
- // Then emit the inline jump table.
- unsigned JTIndex =
- (MCID.Opcode == ARM::BR_JTr)
- ? MI.getOperand(1).getIndex() : MI.getOperand(2).getIndex();
- emitInlineJumpTable(JTIndex);
- return;
- } else if (MCID.Opcode == ARM::BR_JTm) {
- // First emit a ldr pc, [] instruction.
- emitLoadStoreInstruction(MI, ARM::PC);
-
- // Then emit the inline jump table.
- emitInlineJumpTable(MI.getOperand(3).getIndex());
- return;
- }
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- if (MCID.Opcode == ARM::BX_RET || MCID.Opcode == ARM::MOVPCLR)
- // The return register is LR.
- Binary |= II->getRegisterInfo().getEncodingValue(ARM::LR);
- else
- // otherwise, set the return register
- Binary |= getMachineOpValue(MI, 0);
-
- emitWordLE(Binary);
-}
-
-unsigned ARMCodeEmitter::encodeVFPRd(const MachineInstr &MI,
- unsigned OpIdx) const {
- unsigned RegD = MI.getOperand(OpIdx).getReg();
- unsigned Binary = 0;
- bool isSPVFP = ARM::SPRRegClass.contains(RegD);
- RegD = II->getRegisterInfo().getEncodingValue(RegD);
- if (!isSPVFP)
- Binary |= RegD << ARMII::RegRdShift;
- else {
- Binary |= ((RegD & 0x1E) >> 1) << ARMII::RegRdShift;
- Binary |= (RegD & 0x01) << ARMII::D_BitShift;
- }
- return Binary;
-}
-
-unsigned ARMCodeEmitter::encodeVFPRn(const MachineInstr &MI,
- unsigned OpIdx) const {
- unsigned RegN = MI.getOperand(OpIdx).getReg();
- unsigned Binary = 0;
- bool isSPVFP = ARM::SPRRegClass.contains(RegN);
- RegN = II->getRegisterInfo().getEncodingValue(RegN);
- if (!isSPVFP)
- Binary |= RegN << ARMII::RegRnShift;
- else {
- Binary |= ((RegN & 0x1E) >> 1) << ARMII::RegRnShift;
- Binary |= (RegN & 0x01) << ARMII::N_BitShift;
- }
- return Binary;
-}
-
-unsigned ARMCodeEmitter::encodeVFPRm(const MachineInstr &MI,
- unsigned OpIdx) const {
- unsigned RegM = MI.getOperand(OpIdx).getReg();
- unsigned Binary = 0;
- bool isSPVFP = ARM::SPRRegClass.contains(RegM);
- RegM = II->getRegisterInfo().getEncodingValue(RegM);
- if (!isSPVFP)
- Binary |= RegM;
- else {
- Binary |= ((RegM & 0x1E) >> 1);
- Binary |= (RegM & 0x01) << ARMII::M_BitShift;
- }
- return Binary;
-}
-
-void ARMCodeEmitter::emitVFPArithInstruction(const MachineInstr &MI) {
- const MCInstrDesc &MCID = MI.getDesc();
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- unsigned OpIdx = 0;
- assert((Binary & ARMII::D_BitShift) == 0 &&
- (Binary & ARMII::N_BitShift) == 0 &&
- (Binary & ARMII::M_BitShift) == 0 && "VFP encoding bug!");
-
- // Encode Dd / Sd.
- Binary |= encodeVFPRd(MI, OpIdx++);
-
- // If this is a two-address operand, skip it, e.g. FMACD.
- if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
- ++OpIdx;
-
- // Encode Dn / Sn.
- if ((MCID.TSFlags & ARMII::FormMask) == ARMII::VFPBinaryFrm)
- Binary |= encodeVFPRn(MI, OpIdx++);
-
- if (OpIdx == MCID.getNumOperands() ||
- MCID.OpInfo[OpIdx].isPredicate() ||
- MCID.OpInfo[OpIdx].isOptionalDef()) {
- // FCMPEZD etc. has only one operand.
- emitWordLE(Binary);
- return;
- }
-
- // Encode Dm / Sm.
- Binary |= encodeVFPRm(MI, OpIdx);
-
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitVFPConversionInstruction(const MachineInstr &MI) {
- const MCInstrDesc &MCID = MI.getDesc();
- unsigned Form = MCID.TSFlags & ARMII::FormMask;
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- switch (Form) {
- default: break;
- case ARMII::VFPConv1Frm:
- case ARMII::VFPConv2Frm:
- case ARMII::VFPConv3Frm:
- // Encode Dd / Sd.
- Binary |= encodeVFPRd(MI, 0);
- break;
- case ARMII::VFPConv4Frm:
- // Encode Dn / Sn.
- Binary |= encodeVFPRn(MI, 0);
- break;
- case ARMII::VFPConv5Frm:
- // Encode Dm / Sm.
- Binary |= encodeVFPRm(MI, 0);
- break;
- }
-
- switch (Form) {
- default: break;
- case ARMII::VFPConv1Frm:
- // Encode Dm / Sm.
- Binary |= encodeVFPRm(MI, 1);
- break;
- case ARMII::VFPConv2Frm:
- case ARMII::VFPConv3Frm:
- // Encode Dn / Sn.
- Binary |= encodeVFPRn(MI, 1);
- break;
- case ARMII::VFPConv4Frm:
- case ARMII::VFPConv5Frm:
- // Encode Dd / Sd.
- Binary |= encodeVFPRd(MI, 1);
- break;
- }
-
- if (Form == ARMII::VFPConv5Frm)
- // Encode Dn / Sn.
- Binary |= encodeVFPRn(MI, 2);
- else if (Form == ARMII::VFPConv3Frm)
- // Encode Dm / Sm.
- Binary |= encodeVFPRm(MI, 2);
-
- emitWordLE(Binary);
-}
-
-void ARMCodeEmitter::emitVFPLoadStoreInstruction(const MachineInstr &MI) {
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- unsigned OpIdx = 0;
-
- // Encode Dd / Sd.
- Binary |= encodeVFPRd(MI, OpIdx++);
-
- // Encode address base.
- const MachineOperand &Base = MI.getOperand(OpIdx++);
- Binary |= getMachineOpValue(MI, Base) << ARMII::RegRnShift;
-
- // If there is a non-zero immediate offset, encode it.
- if (Base.isReg()) {
- const MachineOperand &Offset = MI.getOperand(OpIdx);
- if (unsigned ImmOffs = ARM_AM::getAM5Offset(Offset.getImm())) {
- if (ARM_AM::getAM5Op(Offset.getImm()) == ARM_AM::add)
- Binary |= 1 << ARMII::U_BitShift;
- Binary |= ImmOffs;
- emitWordLE(Binary);
- return;
- }
- }
-
- // If immediate offset is omitted, default to +0.
- Binary |= 1 << ARMII::U_BitShift;
-
- emitWordLE(Binary);
-}
-
-void
-ARMCodeEmitter::emitVFPLoadStoreMultipleInstruction(const MachineInstr &MI) {
- const MCInstrDesc &MCID = MI.getDesc();
- bool IsUpdating = (MCID.TSFlags & ARMII::IndexModeMask) != 0;
-
- // Part of binary is determined by TableGn.
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- // Set the conditional execution predicate
- Binary |= II->getPredicate(&MI) << ARMII::CondShift;
-
- // Skip operand 0 of an instruction with base register update.
- unsigned OpIdx = 0;
- if (IsUpdating)
- ++OpIdx;
-
- // Set base address operand
- Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
-
- // Set addressing mode by modifying bits U(23) and P(24)
- ARM_AM::AMSubMode Mode = ARM_AM::getLoadStoreMultipleSubMode(MI.getOpcode());
- Binary |= getAddrModeUPBits(ARM_AM::getAM4SubMode(Mode));
-
- // Set bit W(21)
- if (IsUpdating)
- Binary |= 0x1 << ARMII::W_BitShift;
-
- // First register is encoded in Dd.
- Binary |= encodeVFPRd(MI, OpIdx+2);
-
- // Count the number of registers.
- unsigned NumRegs = 1;
- for (unsigned i = OpIdx+3, e = MI.getNumOperands(); i != e; ++i) {
- const MachineOperand &MO = MI.getOperand(i);
- if (!MO.isReg() || MO.isImplicit())
- break;
- ++NumRegs;
- }
- // Bit 8 will be set if <list> is consecutive 64-bit registers (e.g., D0)
- // Otherwise, it will be 0, in the case of 32-bit registers.
- if(Binary & 0x100)
- Binary |= NumRegs * 2;
- else
- Binary |= NumRegs;
-
- emitWordLE(Binary);
-}
-
-unsigned ARMCodeEmitter::encodeNEONRd(const MachineInstr &MI,
- unsigned OpIdx) const {
- unsigned RegD = MI.getOperand(OpIdx).getReg();
- unsigned Binary = 0;
- RegD = II->getRegisterInfo().getEncodingValue(RegD);
- Binary |= (RegD & 0xf) << ARMII::RegRdShift;
- Binary |= ((RegD >> 4) & 1) << ARMII::D_BitShift;
- return Binary;
-}
-
-unsigned ARMCodeEmitter::encodeNEONRn(const MachineInstr &MI,
- unsigned OpIdx) const {
- unsigned RegN = MI.getOperand(OpIdx).getReg();
- unsigned Binary = 0;
- RegN = II->getRegisterInfo().getEncodingValue(RegN);
- Binary |= (RegN & 0xf) << ARMII::RegRnShift;
- Binary |= ((RegN >> 4) & 1) << ARMII::N_BitShift;
- return Binary;
-}
-
-unsigned ARMCodeEmitter::encodeNEONRm(const MachineInstr &MI,
- unsigned OpIdx) const {
- unsigned RegM = MI.getOperand(OpIdx).getReg();
- unsigned Binary = 0;
- RegM = II->getRegisterInfo().getEncodingValue(RegM);
- Binary |= (RegM & 0xf);
- Binary |= ((RegM >> 4) & 1) << ARMII::M_BitShift;
- return Binary;
-}
-
-/// convertNEONDataProcToThumb - Convert the ARM mode encoding for a NEON
-/// data-processing instruction to the corresponding Thumb encoding.
-static unsigned convertNEONDataProcToThumb(unsigned Binary) {
- assert((Binary & 0xfe000000) == 0xf2000000 &&
- "not an ARM NEON data-processing instruction");
- unsigned UBit = (Binary >> 24) & 1;
- return 0xef000000 | (UBit << 28) | (Binary & 0xffffff);
-}
-
-void ARMCodeEmitter::emitNEONLaneInstruction(const MachineInstr &MI) {
- unsigned Binary = getBinaryCodeForInstr(MI);
-
- unsigned RegTOpIdx, RegNOpIdx, LnOpIdx;
- const MCInstrDesc &MCID = MI.getDesc();
- if ((MCID.TSFlags & ARMII::FormMask) == ARMII::NGetLnFrm) {
- RegTOpIdx = 0;
- RegNOpIdx =&
projects / oota-llvm.git / commitdiff