$(TARGET:%=$(ObjDir)/%GenMCCodeEmitter.inc.tmp): \
$(ObjDir)/%GenMCCodeEmitter.inc.tmp: %.td $(ObjDir)/.dir $(LLVM_TBLGEN)
$(Echo) "Building $(<F) MC code emitter with tblgen"
- $(Verb) $(LLVMTableGen) -gen-emitter -o $(call SYSPATH, $@) $<
+ $(Verb) $(LLVMTableGen) -gen-emitter -mc-emitter -o $(call SYSPATH, $@) $<
$(TARGET:%=$(ObjDir)/%GenMCPseudoLowering.inc.tmp): \
$(ObjDir)/%GenMCPseudoLowering.inc.tmp: %.td $(ObjDir)/.dir $(LLVM_TBLGEN)
$(Echo) "Building $(<F) MC Pseudo instruction expander with tblgen"
$(Verb) $(LLVMTableGen) -gen-pseudo-lowering -o $(call SYSPATH, $@) $<
+$(TARGET:%=$(ObjDir)/%GenCodeEmitter.inc.tmp): \
+$(ObjDir)/%GenCodeEmitter.inc.tmp: %.td $(ObjDir)/.dir $(LLVM_TBLGEN)
+ $(Echo) "Building $(<F) code emitter with tblgen"
+ $(Verb) $(LLVMTableGen) -gen-emitter -o $(call SYSPATH, $@) $<
+
$(TARGET:%=$(ObjDir)/%GenDAGISel.inc.tmp): \
$(ObjDir)/%GenDAGISel.inc.tmp : %.td $(ObjDir)/.dir $(LLVM_TBLGEN)
$(Echo) "Building $(<F) DAG instruction selector implementation with tblgen"
LEVEL := ../../..
LIBRARYNAME := llvm_executionengine
-UsedComponents := executionengine mcjit interpreter native
+UsedComponents := executionengine jit interpreter native
UsedOcamlInterfaces := llvm llvm_target
include ../Makefile.ocaml
/* Force the LLVM interpreter and JIT to be linked in. */
void llvm_initialize(void) {
LLVMLinkInInterpreter();
- LLVMLinkInMCJIT();
+ LLVMLinkInJIT();
}
/* unit -> bool */
**Output**: C++ code, implementing the target's CodeEmitter
class by overriding the virtual functions as ``<Target>CodeEmitter::function()``.
-**Usage**: Used to include directly at the end of ``<Target>MCCodeEmitter.cpp``.
+**Usage**: Used to include directly at the end of ``<Target>CodeEmitter.cpp``, and
+with option `-mc-emitter` to be included in ``<Target>MCCodeEmitter.cpp``.
RegisterInfo
------------
#include "BrainF.h"
#include "llvm/Bitcode/ReaderWriter.h"
-#include "llvm/ExecutionEngine/ExecutionEngine.h"
#include "llvm/ExecutionEngine/GenericValue.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/CommandLine.h"
BitWriter
Core
ExecutionEngine
+ JIT
MC
Support
nativecodegen
// Build engine with JIT
llvm::EngineBuilder factory(module);
factory.setEngineKind(llvm::EngineKind::JIT);
+ factory.setAllocateGVsWithCode(false);
factory.setTargetOptions(Opts);
factory.setMCJITMemoryManager(MemMgr);
+ factory.setUseMCJIT(true);
llvm::ExecutionEngine *executionEngine = factory.create();
{
Core
ExecutionEngine
Interpreter
+ JIT
MC
Support
nativecodegen
#include "llvm/IR/Verifier.h"
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/Interpreter.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Instructions.h"
Core
ExecutionEngine
Interpreter
+ JIT
MC
Support
nativecodegen
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/Interpreter.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
// Import result of execution:
outs() << "Result: " << gv.IntVal << "\n";
+ EE->freeMachineCodeForFunction(FooF);
delete EE;
llvm_shutdown();
return 0;
Core
ExecutionEngine
InstCombine
+ JIT
MC
ScalarOpts
Support
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
Core
ExecutionEngine
InstCombine
+ JIT
MC
ScalarOpts
Support
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
Core
ExecutionEngine
InstCombine
+ JIT
MC
ScalarOpts
Support
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
Core
ExecutionEngine
InstCombine
+ JIT
MC
ScalarOpts
Support
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
std::string ErrStr;
ExecutionEngine *NewEngine = EngineBuilder(M)
.setErrorStr(&ErrStr)
+ .setUseMCJIT(true)
.setMCJITMemoryManager(new HelpingMemoryManager(this))
.create();
if (!NewEngine) {
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/ExecutionEngine/MCJIT.h"
#include "llvm/ExecutionEngine/ObjectCache.h"
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
cl::desc("Dump IR from modules to stderr on shutdown"),
cl::init(false));
+ cl::opt<bool> UseMCJIT(
+ "use-mcjit", cl::desc("Use the MCJIT execution engine"),
+ cl::init(true));
+
cl::opt<bool> EnableLazyCompilation(
"enable-lazy-compilation", cl::desc("Enable lazy compilation when using the MCJIT engine"),
cl::init(true));
virtual void dump();
};
+//===----------------------------------------------------------------------===//
+// Helper class for JIT execution engine
+//===----------------------------------------------------------------------===//
+
+class JITHelper : public BaseHelper {
+public:
+ JITHelper(LLVMContext &Context) {
+ // Make the module, which holds all the code.
+ if (!InputIR.empty()) {
+ TheModule = parseInputIR(InputIR, Context);
+ } else {
+ TheModule = new Module("my cool jit", Context);
+ }
+
+ // Create the JIT. This takes ownership of the module.
+ std::string ErrStr;
+ TheExecutionEngine = EngineBuilder(TheModule).setErrorStr(&ErrStr).create();
+ if (!TheExecutionEngine) {
+ fprintf(stderr, "Could not create ExecutionEngine: %s\n", ErrStr.c_str());
+ exit(1);
+ }
+
+ TheFPM = new FunctionPassManager(TheModule);
+
+ // Set up the optimizer pipeline. Start with registering info about how the
+ // target lays out data structures.
+ TheFPM->add(new DataLayout(*TheExecutionEngine->getDataLayout()));
+ // Provide basic AliasAnalysis support for GVN.
+ TheFPM->add(createBasicAliasAnalysisPass());
+ // Promote allocas to registers.
+ TheFPM->add(createPromoteMemoryToRegisterPass());
+ // Do simple "peephole" optimizations and bit-twiddling optzns.
+ TheFPM->add(createInstructionCombiningPass());
+ // Reassociate expressions.
+ TheFPM->add(createReassociatePass());
+ // Eliminate Common SubExpressions.
+ TheFPM->add(createGVNPass());
+ // Simplify the control flow graph (deleting unreachable blocks, etc).
+ TheFPM->add(createCFGSimplificationPass());
+
+ TheFPM->doInitialization();
+ }
+
+ virtual ~JITHelper() {
+ if (TheFPM)
+ delete TheFPM;
+ if (TheExecutionEngine)
+ delete TheExecutionEngine;
+ }
+
+ virtual Function *getFunction(const std::string FnName) {
+ assert(TheModule);
+ return TheModule->getFunction(FnName);
+ }
+
+ virtual Module *getModuleForNewFunction() {
+ assert(TheModule);
+ return TheModule;
+ }
+
+ virtual void *getPointerToFunction(Function* F) {
+ assert(TheExecutionEngine);
+ return TheExecutionEngine->getPointerToFunction(F);
+ }
+
+ virtual void *getPointerToNamedFunction(const std::string &Name) {
+ return TheExecutionEngine->getPointerToNamedFunction(Name);
+ }
+
+ virtual void runFPM(Function &F) {
+ assert(TheFPM);
+ TheFPM->run(F);
+ }
+
+ virtual void closeCurrentModule() {
+ // This should never be called for JIT
+ assert(false);
+ }
+
+ virtual void dump() {
+ assert(TheModule);
+ TheModule->dump();
+ }
+
+private:
+ Module *TheModule;
+ ExecutionEngine *TheExecutionEngine;
+ FunctionPassManager *TheFPM;
+};
+
//===----------------------------------------------------------------------===//
// MCJIT helper class
//===----------------------------------------------------------------------===//
std::string ErrStr;
ExecutionEngine *EE = EngineBuilder(M)
.setErrorStr(&ErrStr)
+ .setUseMCJIT(true)
.setMCJITMemoryManager(new HelpingMemoryManager(this))
.create();
if (!EE) {
Value *OperandV = Operand->Codegen();
if (OperandV == 0) return 0;
Function *F;
- F = TheHelper->getFunction(
- MakeLegalFunctionName(std::string("unary") + Opcode));
+ if (UseMCJIT)
+ F = TheHelper->getFunction(MakeLegalFunctionName(std::string("unary")+Opcode));
+ else
+ F = TheHelper->getFunction(std::string("unary")+Opcode);
if (F == 0)
return ErrorV("Unknown unary operator");
// If it wasn't a builtin binary operator, it must be a user defined one. Emit
// a call to it.
Function *F;
- F = TheHelper->getFunction(MakeLegalFunctionName(std::string("binary")+Op));
+ if (UseMCJIT)
+ F = TheHelper->getFunction(MakeLegalFunctionName(std::string("binary")+Op));
+ else
+ F = TheHelper->getFunction(std::string("binary")+Op);
assert(F && "binary operator not found!");
Value *Ops[] = { L, R };
Doubles, false);
std::string FnName;
- FnName = MakeLegalFunctionName(Name);
+ if (UseMCJIT)
+ FnName = MakeLegalFunctionName(Name);
+ else
+ FnName = Name;
Module* M = TheHelper->getModuleForNewFunction();
Function *F = Function::Create(FT, Function::ExternalLinkage, FnName, M);
// Validate the generated code, checking for consistency.
verifyFunction(*TheFunction);
+ // Optimize the function.
+ if (!UseMCJIT)
+ TheHelper->runFPM(*TheFunction);
+
return TheFunction;
}
static void HandleDefinition() {
if (FunctionAST *F = ParseDefinition()) {
- if (EnableLazyCompilation)
+ if (UseMCJIT && EnableLazyCompilation)
TheHelper->closeCurrentModule();
Function *LF = F->Codegen();
if (LF && VerboseOutput) {
int main(int argc, char **argv) {
InitializeNativeTarget();
- InitializeNativeTargetAsmPrinter();
- InitializeNativeTargetAsmParser();
+ if (UseMCJIT) {
+ InitializeNativeTargetAsmPrinter();
+ InitializeNativeTargetAsmParser();
+ }
LLVMContext &Context = getGlobalContext();
cl::ParseCommandLineOptions(argc, argv,
BinopPrecedence['*'] = 40; // highest.
// Make the Helper, which holds all the code.
- TheHelper = new MCJITHelper(Context);
+ if (UseMCJIT)
+ TheHelper = new MCJITHelper(Context);
+ else
+ TheHelper = new JITHelper(Context);
// Prime the first token.
if (!SuppressPrompts)
std::string ErrStr;
ExecutionEngine *NewEngine = EngineBuilder(OpenModule)
.setErrorStr(&ErrStr)
+ .setUseMCJIT(true)
.setMCJITMemoryManager(new HelpingMemoryManager(this))
.create();
if (!NewEngine) {
#include "llvm/Analysis/Passes.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
std::string ErrStr;
ExecutionEngine *NewEngine = EngineBuilder(M)
.setErrorStr(&ErrStr)
+ .setUseMCJIT(true)
.setMCJITMemoryManager(new HelpingMemoryManager(this))
.create();
if (!NewEngine) {
Core
ExecutionEngine
Interpreter
+ JIT
Support
nativecodegen
)
#include "llvm/ExecutionEngine/GenericValue.h"
#include "llvm/ExecutionEngine/Interpreter.h"
+#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Instructions.h"
* @{
*/
+void LLVMLinkInJIT(void);
void LLVMLinkInMCJIT(void);
void LLVMLinkInInterpreter(void);
--- /dev/null
+//===-- llvm/CodeGen/JITCodeEmitter.h - Code emission ----------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines an abstract interface that is used by the machine code
+// emission framework to output the code. This allows machine code emission to
+// be separated from concerns such as resolution of call targets, and where the
+// machine code will be written (memory or disk, f.e.).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_JITCODEEMITTER_H
+#define LLVM_CODEGEN_JITCODEEMITTER_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/MachineCodeEmitter.h"
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/MathExtras.h"
+#include <string>
+
+namespace llvm {
+
+class MachineBasicBlock;
+class MachineConstantPool;
+class MachineJumpTableInfo;
+class MachineFunction;
+class MachineModuleInfo;
+class MachineRelocation;
+class Value;
+class GlobalValue;
+class Function;
+
+/// JITCodeEmitter - This class defines two sorts of methods: those for
+/// emitting the actual bytes of machine code, and those for emitting auxiliary
+/// structures, such as jump tables, relocations, etc.
+///
+/// Emission of machine code is complicated by the fact that we don't (in
+/// general) know the size of the machine code that we're about to emit before
+/// we emit it. As such, we preallocate a certain amount of memory, and set the
+/// BufferBegin/BufferEnd pointers to the start and end of the buffer. As we
+/// emit machine instructions, we advance the CurBufferPtr to indicate the
+/// location of the next byte to emit. In the case of a buffer overflow (we
+/// need to emit more machine code than we have allocated space for), the
+/// CurBufferPtr will saturate to BufferEnd and ignore stores. Once the entire
+/// function has been emitted, the overflow condition is checked, and if it has
+/// occurred, more memory is allocated, and we reemit the code into it.
+///
+class JITCodeEmitter : public MachineCodeEmitter {
+ void anchor() override;
+public:
+ virtual ~JITCodeEmitter() {}
+
+ /// startFunction - This callback is invoked when the specified function is
+ /// about to be code generated. This initializes the BufferBegin/End/Ptr
+ /// fields.
+ ///
+ void startFunction(MachineFunction &F) override = 0;
+
+ /// finishFunction - This callback is invoked when the specified function has
+ /// finished code generation. If a buffer overflow has occurred, this method
+ /// returns true (the callee is required to try again), otherwise it returns
+ /// false.
+ ///
+ bool finishFunction(MachineFunction &F) override = 0;
+
+ /// allocIndirectGV - Allocates and fills storage for an indirect
+ /// GlobalValue, and returns the address.
+ virtual void *allocIndirectGV(const GlobalValue *GV,
+ const uint8_t *Buffer, size_t Size,
+ unsigned Alignment) = 0;
+
+ /// emitByte - This callback is invoked when a byte needs to be written to the
+ /// output stream.
+ ///
+ void emitByte(uint8_t B) {
+ if (CurBufferPtr != BufferEnd)
+ *CurBufferPtr++ = B;
+ }
+
+ /// emitWordLE - This callback is invoked when a 32-bit word needs to be
+ /// written to the output stream in little-endian format.
+ ///
+ void emitWordLE(uint32_t W) {
+ if (4 <= BufferEnd-CurBufferPtr) {
+ *CurBufferPtr++ = (uint8_t)(W >> 0);
+ *CurBufferPtr++ = (uint8_t)(W >> 8);
+ *CurBufferPtr++ = (uint8_t)(W >> 16);
+ *CurBufferPtr++ = (uint8_t)(W >> 24);
+ } else {
+ CurBufferPtr = BufferEnd;
+ }
+ }
+
+ /// emitWordBE - This callback is invoked when a 32-bit word needs to be
+ /// written to the output stream in big-endian format.
+ ///
+ void emitWordBE(uint32_t W) {
+ if (4 <= BufferEnd-CurBufferPtr) {
+ *CurBufferPtr++ = (uint8_t)(W >> 24);
+ *CurBufferPtr++ = (uint8_t)(W >> 16);
+ *CurBufferPtr++ = (uint8_t)(W >> 8);
+ *CurBufferPtr++ = (uint8_t)(W >> 0);
+ } else {
+ CurBufferPtr = BufferEnd;
+ }
+ }
+
+ /// emitDWordLE - This callback is invoked when a 64-bit word needs to be
+ /// written to the output stream in little-endian format.
+ ///
+ void emitDWordLE(uint64_t W) {
+ if (8 <= BufferEnd-CurBufferPtr) {
+ *CurBufferPtr++ = (uint8_t)(W >> 0);
+ *CurBufferPtr++ = (uint8_t)(W >> 8);
+ *CurBufferPtr++ = (uint8_t)(W >> 16);
+ *CurBufferPtr++ = (uint8_t)(W >> 24);
+ *CurBufferPtr++ = (uint8_t)(W >> 32);
+ *CurBufferPtr++ = (uint8_t)(W >> 40);
+ *CurBufferPtr++ = (uint8_t)(W >> 48);
+ *CurBufferPtr++ = (uint8_t)(W >> 56);
+ } else {
+ CurBufferPtr = BufferEnd;
+ }
+ }
+
+ /// emitDWordBE - This callback is invoked when a 64-bit word needs to be
+ /// written to the output stream in big-endian format.
+ ///
+ void emitDWordBE(uint64_t W) {
+ if (8 <= BufferEnd-CurBufferPtr) {
+ *CurBufferPtr++ = (uint8_t)(W >> 56);
+ *CurBufferPtr++ = (uint8_t)(W >> 48);
+ *CurBufferPtr++ = (uint8_t)(W >> 40);
+ *CurBufferPtr++ = (uint8_t)(W >> 32);
+ *CurBufferPtr++ = (uint8_t)(W >> 24);
+ *CurBufferPtr++ = (uint8_t)(W >> 16);
+ *CurBufferPtr++ = (uint8_t)(W >> 8);
+ *CurBufferPtr++ = (uint8_t)(W >> 0);
+ } else {
+ CurBufferPtr = BufferEnd;
+ }
+ }
+
+ /// emitAlignment - Move the CurBufferPtr pointer up to the specified
+ /// alignment (saturated to BufferEnd of course).
+ void emitAlignment(unsigned Alignment) {
+ if (Alignment == 0) Alignment = 1;
+ uint8_t *NewPtr = (uint8_t*)RoundUpToAlignment((uintptr_t)CurBufferPtr,
+ Alignment);
+ CurBufferPtr = std::min(NewPtr, BufferEnd);
+ }
+
+ /// emitAlignmentWithFill - Similar to emitAlignment, except that the
+ /// extra bytes are filled with the provided byte.
+ void emitAlignmentWithFill(unsigned Alignment, uint8_t Fill) {
+ if (Alignment == 0) Alignment = 1;
+ uint8_t *NewPtr = (uint8_t*)RoundUpToAlignment((uintptr_t)CurBufferPtr,
+ Alignment);
+ // Fail if we don't have room.
+ if (NewPtr > BufferEnd) {
+ CurBufferPtr = BufferEnd;
+ return;
+ }
+ while (CurBufferPtr < NewPtr) {
+ *CurBufferPtr++ = Fill;
+ }
+ }
+
+ /// emitULEB128Bytes - This callback is invoked when a ULEB128 needs to be
+ /// written to the output stream.
+ void emitULEB128Bytes(uint64_t Value, unsigned PadTo = 0) {
+ do {
+ uint8_t Byte = Value & 0x7f;
+ Value >>= 7;
+ if (Value || PadTo != 0) Byte |= 0x80;
+ emitByte(Byte);
+ } while (Value);
+
+ if (PadTo) {
+ do {
+ uint8_t Byte = (PadTo > 1) ? 0x80 : 0x0;
+ emitByte(Byte);
+ } while (--PadTo);
+ }
+ }
+
+ /// emitSLEB128Bytes - This callback is invoked when a SLEB128 needs to be
+ /// written to the output stream.
+ void emitSLEB128Bytes(int64_t Value) {
+ int32_t Sign = Value >> (8 * sizeof(Value) - 1);
+ bool IsMore;
+
+ do {
+ uint8_t Byte = Value & 0x7f;
+ Value >>= 7;
+ IsMore = Value != Sign || ((Byte ^ Sign) & 0x40) != 0;
+ if (IsMore) Byte |= 0x80;
+ emitByte(Byte);
+ } while (IsMore);
+ }
+
+ /// emitString - This callback is invoked when a String needs to be
+ /// written to the output stream.
+ void emitString(const std::string &String) {
+ for (size_t i = 0, N = String.size(); i < N; ++i) {
+ uint8_t C = String[i];
+ emitByte(C);
+ }
+ emitByte(0);
+ }
+
+ /// emitInt32 - Emit a int32 directive.
+ void emitInt32(uint32_t Value) {
+ if (4 <= BufferEnd-CurBufferPtr) {
+ *((uint32_t*)CurBufferPtr) = Value;
+ CurBufferPtr += 4;
+ } else {
+ CurBufferPtr = BufferEnd;
+ }
+ }
+
+ /// emitInt64 - Emit a int64 directive.
+ void emitInt64(uint64_t Value) {
+ if (8 <= BufferEnd-CurBufferPtr) {
+ *((uint64_t*)CurBufferPtr) = Value;
+ CurBufferPtr += 8;
+ } else {
+ CurBufferPtr = BufferEnd;
+ }
+ }
+
+ /// emitInt32At - Emit the Int32 Value in Addr.
+ void emitInt32At(uintptr_t *Addr, uintptr_t Value) {
+ if (Addr >= (uintptr_t*)BufferBegin && Addr < (uintptr_t*)BufferEnd)
+ (*(uint32_t*)Addr) = (uint32_t)Value;
+ }
+
+ /// emitInt64At - Emit the Int64 Value in Addr.
+ void emitInt64At(uintptr_t *Addr, uintptr_t Value) {
+ if (Addr >= (uintptr_t*)BufferBegin && Addr < (uintptr_t*)BufferEnd)
+ (*(uint64_t*)Addr) = (uint64_t)Value;
+ }
+
+
+ /// emitLabel - Emits a label
+ void emitLabel(MCSymbol *Label) override = 0;
+
+ /// allocateSpace - Allocate a block of space in the current output buffer,
+ /// returning null (and setting conditions to indicate buffer overflow) on
+ /// failure. Alignment is the alignment in bytes of the buffer desired.
+ void *allocateSpace(uintptr_t Size, unsigned Alignment) override {
+ emitAlignment(Alignment);
+ void *Result;
+
+ // Check for buffer overflow.
+ if (Size >= (uintptr_t)(BufferEnd-CurBufferPtr)) {
+ CurBufferPtr = BufferEnd;
+ Result = nullptr;
+ } else {
+ // Allocate the space.
+ Result = CurBufferPtr;
+ CurBufferPtr += Size;
+ }
+
+ return Result;
+ }
+
+ /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace,
+ /// this method does not allocate memory in the current output buffer,
+ /// because a global may live longer than the current function.
+ virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment) = 0;
+
+ /// StartMachineBasicBlock - This should be called by the target when a new
+ /// basic block is about to be emitted. This way the MCE knows where the
+ /// start of the block is, and can implement getMachineBasicBlockAddress.
+ void StartMachineBasicBlock(MachineBasicBlock *MBB) override = 0;
+
+ /// getCurrentPCValue - This returns the address that the next emitted byte
+ /// will be output to.
+ ///
+ uintptr_t getCurrentPCValue() const override {
+ return (uintptr_t)CurBufferPtr;
+ }
+
+ /// getCurrentPCOffset - Return the offset from the start of the emitted
+ /// buffer that we are currently writing to.
+ uintptr_t getCurrentPCOffset() const override {
+ return CurBufferPtr-BufferBegin;
+ }
+
+ /// earlyResolveAddresses - True if the code emitter can use symbol addresses
+ /// during code emission time. The JIT is capable of doing this because it
+ /// creates jump tables or constant pools in memory on the fly while the
+ /// object code emitters rely on a linker to have real addresses and should
+ /// use relocations instead.
+ bool earlyResolveAddresses() const override { return true; }
+
+ /// addRelocation - Whenever a relocatable address is needed, it should be
+ /// noted with this interface.
+ void addRelocation(const MachineRelocation &MR) override = 0;
+
+ /// FIXME: These should all be handled with relocations!
+
+ /// getConstantPoolEntryAddress - Return the address of the 'Index' entry in
+ /// the constant pool that was last emitted with the emitConstantPool method.
+ ///
+ uintptr_t getConstantPoolEntryAddress(unsigned Index) const override = 0;
+
+ /// getJumpTableEntryAddress - Return the address of the jump table with index
+ /// 'Index' in the function that last called initJumpTableInfo.
+ ///
+ uintptr_t getJumpTableEntryAddress(unsigned Index) const override = 0;
+
+ /// getMachineBasicBlockAddress - Return the address of the specified
+ /// MachineBasicBlock, only usable after the label for the MBB has been
+ /// emitted.
+ ///
+ uintptr_t
+ getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override = 0;
+
+ /// getLabelAddress - Return the address of the specified Label, only usable
+ /// after the Label has been emitted.
+ ///
+ uintptr_t getLabelAddress(MCSymbol *Label) const override = 0;
+
+ /// Specifies the MachineModuleInfo object. This is used for exception handling
+ /// purposes.
+ void setModuleInfo(MachineModuleInfo* Info) override = 0;
+
+ /// getLabelLocations - Return the label locations map of the label IDs to
+ /// their address.
+ virtual DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() {
+ return nullptr;
+ }
+};
+
+} // End llvm namespace
+
+#endif
// To avoid having libexecutionengine depend on the JIT and interpreter
// libraries, the execution engine implementations set these functions to ctor
// pointers at startup time if they are linked in.
+ static ExecutionEngine *(*JITCtor)(
+ Module *M,
+ std::string *ErrorStr,
+ JITMemoryManager *JMM,
+ bool GVsWithCode,
+ TargetMachine *TM);
static ExecutionEngine *(*MCJITCtor)(
Module *M,
std::string *ErrorStr,
/// getFunctionAddress instead.
virtual void *getPointerToFunction(Function *F) = 0;
+ /// getPointerToBasicBlock - The different EE's represent basic blocks in
+ /// different ways. Return the representation for a blockaddress of the
+ /// specified block.
+ ///
+ /// This function will not be implemented for the MCJIT execution engine.
+ virtual void *getPointerToBasicBlock(BasicBlock *BB) = 0;
+
/// getPointerToFunctionOrStub - If the specified function has been
/// code-gen'd, return a pointer to the function. If not, compile it, or use
/// a stub to implement lazy compilation if available. See
void InitializeMemory(const Constant *Init, void *Addr);
+ /// recompileAndRelinkFunction - This method is used to force a function which
+ /// has already been compiled to be compiled again, possibly after it has been
+ /// modified. Then the entry to the old copy is overwritten with a branch to
+ /// the new copy. If there was no old copy, this acts just like
+ /// VM::getPointerToFunction().
+ virtual void *recompileAndRelinkFunction(Function *F) = 0;
+
+ /// freeMachineCodeForFunction - Release memory in the ExecutionEngine
+ /// corresponding to the machine code emitted to execute this function, useful
+ /// for garbage-collecting generated code.
+ virtual void freeMachineCodeForFunction(Function *F) = 0;
+
/// getOrEmitGlobalVariable - Return the address of the specified global
/// variable, possibly emitting it to memory if needed. This is used by the
/// Emitter.
CodeGenOpt::Level OptLevel;
RTDyldMemoryManager *MCJMM;
JITMemoryManager *JMM;
+ bool AllocateGVsWithCode;
TargetOptions Options;
Reloc::Model RelocModel;
CodeModel::Model CMModel;
std::string MArch;
std::string MCPU;
SmallVector<std::string, 4> MAttrs;
+ bool UseMCJIT;
bool VerifyModules;
/// InitEngine - Does the common initialization of default options.
return *this;
}
+ /// setAllocateGVsWithCode - Sets whether global values should be allocated
+ /// into the same buffer as code. For most applications this should be set
+ /// to false. Allocating globals with code breaks freeMachineCodeForFunction
+ /// and is probably unsafe and bad for performance. However, we have clients
+ /// who depend on this behavior, so we must support it. This option defaults
+ /// to false so that users of the new API can safely use the new memory
+ /// manager and free machine code.
+ EngineBuilder &setAllocateGVsWithCode(bool a) {
+ AllocateGVsWithCode = a;
+ return *this;
+ }
+
/// setMArch - Override the architecture set by the Module's triple.
EngineBuilder &setMArch(StringRef march) {
MArch.assign(march.begin(), march.end());
return *this;
}
+ /// setUseMCJIT - Set whether the MC-JIT implementation should be used
+ /// (experimental).
+ EngineBuilder &setUseMCJIT(bool Value) {
+ UseMCJIT = Value;
+ return *this;
+ }
+
/// setVerifyModules - Set whether the JIT implementation should verify
/// IR modules during compilation.
EngineBuilder &setVerifyModules(bool Verify) {
--- /dev/null
+//===-- JIT.h - Abstract Execution Engine Interface -------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file forces the JIT to link in on certain operating systems.
+// (Windows).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_JIT_H
+#define LLVM_EXECUTIONENGINE_JIT_H
+
+#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include <cstdlib>
+
+extern "C" void LLVMLinkInJIT();
+
+namespace {
+ struct ForceJITLinking {
+ ForceJITLinking() {
+ // We must reference JIT in such a way that compilers will not
+ // delete it all as dead code, even with whole program optimization,
+ // yet is effectively a NO-OP. As the compiler isn't smart enough
+ // to know that getenv() never returns -1, this will do the job.
+ if (std::getenv("bar") != (char*) -1)
+ return;
+
+ LLVMLinkInJIT();
+ }
+ } ForceJITLinking;
+}
+
+#endif
--- /dev/null
+//===- Target/TargetJITInfo.h - Target Information for JIT ------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file exposes an abstract interface used by the Just-In-Time code
+// generator to perform target-specific activities, such as emitting stubs. If
+// a TargetMachine supports JIT code generation, it should provide one of these
+// objects through the getJITInfo() method.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TARGET_TARGETJITINFO_H
+#define LLVM_TARGET_TARGETJITINFO_H
+
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <cassert>
+
+namespace llvm {
+ class Function;
+ class GlobalValue;
+ class JITCodeEmitter;
+ class MachineRelocation;
+
+ /// TargetJITInfo - Target specific information required by the Just-In-Time
+ /// code generator.
+ class TargetJITInfo {
+ virtual void anchor();
+ public:
+ virtual ~TargetJITInfo() {}
+
+ /// replaceMachineCodeForFunction - Make it so that calling the function
+ /// whose machine code is at OLD turns into a call to NEW, perhaps by
+ /// overwriting OLD with a branch to NEW. This is used for self-modifying
+ /// code.
+ ///
+ virtual void replaceMachineCodeForFunction(void *Old, void *New) = 0;
+
+ /// emitGlobalValueIndirectSym - Use the specified JITCodeEmitter object
+ /// to emit an indirect symbol which contains the address of the specified
+ /// ptr.
+ virtual void *emitGlobalValueIndirectSym(const GlobalValue* GV, void *ptr,
+ JITCodeEmitter &JCE) {
+ llvm_unreachable("This target doesn't implement "
+ "emitGlobalValueIndirectSym!");
+ }
+
+ /// Records the required size and alignment for a call stub in bytes.
+ struct StubLayout {
+ size_t Size;
+ size_t Alignment;
+ };
+ /// Returns the maximum size and alignment for a call stub on this target.
+ virtual StubLayout getStubLayout() {
+ llvm_unreachable("This target doesn't implement getStubLayout!");
+ }
+
+ /// emitFunctionStub - Use the specified JITCodeEmitter object to emit a
+ /// small native function that simply calls the function at the specified
+ /// address. The JITCodeEmitter must already have storage allocated for the
+ /// stub. Return the address of the resultant function, which may have been
+ /// aligned from the address the JCE was set up to emit at.
+ virtual void *emitFunctionStub(const Function* F, void *Target,
+ JITCodeEmitter &JCE) {
+ llvm_unreachable("This target doesn't implement emitFunctionStub!");
+ }
+
+ /// getPICJumpTableEntry - Returns the value of the jumptable entry for the
+ /// specific basic block.
+ virtual uintptr_t getPICJumpTableEntry(uintptr_t BB, uintptr_t JTBase) {
+ llvm_unreachable("This target doesn't implement getPICJumpTableEntry!");
+ }
+
+ /// LazyResolverFn - This typedef is used to represent the function that
+ /// unresolved call points should invoke. This is a target specific
+ /// function that knows how to walk the stack and find out which stub the
+ /// call is coming from.
+ typedef void (*LazyResolverFn)();
+
+ /// JITCompilerFn - This typedef is used to represent the JIT function that
+ /// lazily compiles the function corresponding to a stub. The JIT keeps
+ /// track of the mapping between stubs and LLVM Functions, the target
+ /// provides the ability to figure out the address of a stub that is called
+ /// by the LazyResolverFn.
+ typedef void* (*JITCompilerFn)(void *);
+
+ /// getLazyResolverFunction - This method is used to initialize the JIT,
+ /// giving the target the function that should be used to compile a
+ /// function, and giving the JIT the target function used to do the lazy
+ /// resolving.
+ virtual LazyResolverFn getLazyResolverFunction(JITCompilerFn) {
+ llvm_unreachable("Not implemented for this target!");
+ }
+
+ /// relocate - Before the JIT can run a block of code that has been emitted,
+ /// it must rewrite the code to contain the actual addresses of any
+ /// referenced global symbols.
+ virtual void relocate(void *Function, MachineRelocation *MR,
+ unsigned NumRelocs, unsigned char* GOTBase) {
+ assert(NumRelocs == 0 && "This target does not have relocations!");
+ }
+
+ /// allocateThreadLocalMemory - Each target has its own way of
+ /// handling thread local variables. This method returns a value only
+ /// meaningful to the target.
+ virtual char* allocateThreadLocalMemory(size_t size) {
+ llvm_unreachable("This target does not implement thread local storage!");
+ }
+
+ /// needsGOT - Allows a target to specify that it would like the
+ /// JIT to manage a GOT for it.
+ bool needsGOT() const { return useGOT; }
+
+ /// hasCustomConstantPool - Allows a target to specify that constant
+ /// pool address resolution is handled by the target.
+ virtual bool hasCustomConstantPool() const { return false; }
+
+ /// hasCustomJumpTables - Allows a target to specify that jumptables
+ /// are emitted by the target.
+ virtual bool hasCustomJumpTables() const { return false; }
+
+ /// allocateSeparateGVMemory - If true, globals should be placed in
+ /// separately allocated heap memory rather than in the same
+ /// code memory allocated by JITCodeEmitter.
+ virtual bool allocateSeparateGVMemory() const { return false; }
+ protected:
+ bool useGOT;
+ };
+} // End llvm namespace
+
+#endif
return UseUnderscoreLongJmp;
}
+ /// Return whether the target can generate code for jump tables.
+ bool supportJumpTables() const {
+ return SupportJumpTables;
+ }
+
/// Return integer threshold on number of blocks to use jump tables rather
/// than if sequence.
int getMinimumJumpTableEntries() const {
UseUnderscoreLongJmp = Val;
}
+ /// Indicate whether the target can generate code for jump tables.
+ void setSupportJumpTables(bool Val) {
+ SupportJumpTables = Val;
+ }
+
/// Indicate the number of blocks to generate jump tables rather than if
/// sequence.
void setMinimumJumpTableEntries(int Val) {
/// Defaults to false.
bool UseUnderscoreLongJmp;
+ /// Whether the target can generate code for jumptables. If it's not true,
+ /// then each jumptable must be lowered into if-then-else's.
+ bool SupportJumpTables;
+
/// Number of blocks threshold to use jump tables.
int MinimumJumpTableEntries;
namespace llvm {
class InstrItineraryData;
+class JITCodeEmitter;
class GlobalValue;
class Mangler;
class MCAsmInfo;
class TargetLibraryInfo;
class TargetFrameLowering;
class TargetIntrinsicInfo;
+class TargetJITInfo;
class TargetLowering;
class TargetPassConfig;
class TargetRegisterInfo;
virtual const TargetSubtargetInfo *getSubtargetImpl() const {
return nullptr;
}
+ TargetSubtargetInfo *getSubtargetImpl() {
+ const TargetMachine *TM = this;
+ return const_cast<TargetSubtargetInfo *>(TM->getSubtargetImpl());
+ }
/// getSubtarget - This method returns a pointer to the specified type of
/// TargetSubtargetInfo. In debug builds, it verifies that the object being
return true;
}
+ /// addPassesToEmitMachineCode - Add passes to the specified pass manager to
+ /// get machine code emitted. This uses a JITCodeEmitter object to handle
+ /// actually outputting the machine code and resolving things like the address
+ /// of functions. This method returns true if machine code emission is
+ /// not supported.
+ ///
+ virtual bool addPassesToEmitMachineCode(PassManagerBase &,
+ JITCodeEmitter &,
+ bool /*DisableVerify*/ = true) {
+ return true;
+ }
+
/// addPassesToEmitMC - Add passes to the specified pass manager to get
/// machine code emitted with the MCJIT. This method returns true if machine
/// code is not supported. It fills the MCContext Ctx pointer which can be
AnalysisID StartAfter = nullptr,
AnalysisID StopAfter = nullptr) override;
+ /// addPassesToEmitMachineCode - Add passes to the specified pass manager to
+ /// get machine code emitted. This uses a JITCodeEmitter object to handle
+ /// actually outputting the machine code and resolving things like the address
+ /// of functions. This method returns true if machine code emission is
+ /// not supported.
+ ///
+ bool addPassesToEmitMachineCode(PassManagerBase &PM, JITCodeEmitter &MCE,
+ bool DisableVerify = true) override;
+
/// addPassesToEmitMC - Add passes to the specified pass manager to get
/// machine code emitted with the MCJIT. This method returns true if machine
/// code is not supported. It fills the MCContext Ctx pointer which can be
///
bool addPassesToEmitMC(PassManagerBase &PM, MCContext *&Ctx,
raw_ostream &OS, bool DisableVerify = true) override;
+
+ /// addCodeEmitter - This pass should be overridden by the target to add a
+ /// code emitter, if supported. If this is not supported, 'true' should be
+ /// returned.
+ virtual bool addCodeEmitter(PassManagerBase &,
+ JITCodeEmitter &) {
+ return true;
+ }
};
} // End llvm namespace
class SUnit;
class TargetFrameLowering;
class TargetInstrInfo;
+class TargetJITInfo;
class TargetLowering;
class TargetRegisterClass;
class TargetRegisterInfo;
///
virtual const TargetRegisterInfo *getRegisterInfo() const { return nullptr; }
+ /// getJITInfo - If this target supports a JIT, return information for it,
+ /// otherwise return null.
+ ///
+ virtual TargetJITInfo *getJITInfo() { return nullptr; }
+
/// getInstrItineraryData - Returns instruction itinerary data for the target
/// or specific subtarget.
///
bool BasicTTI::shouldBuildLookupTables() const {
const TargetLoweringBase *TLI = getTLI();
- return TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
- TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
+ return TLI->supportJumpTables() &&
+ (TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
+ TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other));
}
bool BasicTTI::haveFastSqrt(Type *Ty) const {
InlineSpiller.cpp
InterferenceCache.cpp
IntrinsicLowering.cpp
+ JITCodeEmitter.cpp
JumpInstrTables.cpp
LLVMTargetMachine.cpp
LatencyPriorityQueue.cpp
--- /dev/null
+//===-- llvm/CodeGen/JITCodeEmitter.cpp - Code emission --------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/JITCodeEmitter.h"
+
+using namespace llvm;
+
+void JITCodeEmitter::anchor() { }
return false;
}
+/// addPassesToEmitMachineCode - Add passes to the specified pass manager to
+/// get machine code emitted. This uses a JITCodeEmitter object to handle
+/// actually outputting the machine code and resolving things like the address
+/// of functions. This method should return true if machine code emission is
+/// not supported.
+///
+bool LLVMTargetMachine::addPassesToEmitMachineCode(PassManagerBase &PM,
+ JITCodeEmitter &JCE,
+ bool DisableVerify) {
+ // Add common CodeGen passes.
+ MCContext *Context = addPassesToGenerateCode(this, PM, DisableVerify, nullptr,
+ nullptr);
+ if (!Context)
+ return true;
+
+ addCodeEmitter(PM, JCE);
+
+ return false; // success!
+}
+
/// addPassesToEmitMC - Add passes to the specified pass manager to get
/// machine code emitted with the MCJIT. This method returns true if machine
/// code is not supported. It fills the MCContext Ctx pointer which can be
}
static inline bool areJTsAllowed(const TargetLowering &TLI) {
- return TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
- TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
+ return TLI.supportJumpTables() &&
+ (TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
+ TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other));
}
static APInt ComputeRange(const APInt &First, const APInt &Last) {
PrefLoopAlignment = 0;
MinStackArgumentAlignment = 1;
InsertFencesForAtomic = false;
+ SupportJumpTables = true;
MinimumJumpTableEntries = 4;
InitLibcallNames(LibcallRoutineNames, Triple(TM.getTargetTriple()));
)
add_subdirectory(Interpreter)
+add_subdirectory(JIT)
add_subdirectory(MCJIT)
add_subdirectory(RuntimeDyld)
void ObjectBuffer::anchor() {}
void ObjectBufferStream::anchor() {}
+ExecutionEngine *(*ExecutionEngine::JITCtor)(
+ Module *M,
+ std::string *ErrorStr,
+ JITMemoryManager *JMM,
+ bool GVsWithCode,
+ TargetMachine *TM) = nullptr;
ExecutionEngine *(*ExecutionEngine::MCJITCtor)(
Module *M,
std::string *ErrorStr,
MCJMM = nullptr;
JMM = nullptr;
Options = TargetOptions();
+ AllocateGVsWithCode = false;
RelocModel = Reloc::Default;
CMModel = CodeModel::JITDefault;
+ UseMCJIT = false;
// IR module verification is enabled by default in debug builds, and disabled
// by default in release builds.
return nullptr;
}
}
+
+ if (MCJMM && ! UseMCJIT) {
+ if (ErrorStr)
+ *ErrorStr =
+ "Cannot create a legacy JIT with a runtime dyld memory "
+ "manager.";
+ return nullptr;
+ }
// Unless the interpreter was explicitly selected or the JIT is not linked,
// try making a JIT.
}
ExecutionEngine *EE = nullptr;
- if (ExecutionEngine::MCJITCtor)
+ if (UseMCJIT && ExecutionEngine::MCJITCtor)
EE = ExecutionEngine::MCJITCtor(M, ErrorStr, MCJMM ? MCJMM : JMM,
TheTM.release());
+ else if (ExecutionEngine::JITCtor)
+ EE = ExecutionEngine::JITCtor(M, ErrorStr, JMM,
+ AllocateGVsWithCode, TheTM.release());
if (EE) {
EE->setVerifyModules(VerifyModules);
return nullptr;
}
- if ((WhichEngine & EngineKind::JIT) && !ExecutionEngine::MCJITCtor) {
+ if ((WhichEngine & EngineKind::JIT) && !ExecutionEngine::JITCtor &&
+ !ExecutionEngine::MCJITCtor) {
if (ErrorStr)
*ErrorStr = "JIT has not been linked in.";
}
Result = PTOGV(getPointerToFunctionOrStub(const_cast<Function*>(F)));
else if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(C))
Result = PTOGV(getOrEmitGlobalVariable(const_cast<GlobalVariable*>(GV)));
+ else if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
+ Result = PTOGV(getPointerToBasicBlock(const_cast<BasicBlock*>(
+ BA->getBasicBlock())));
else
llvm_unreachable("Unknown constant pointer type!");
break;
EngineBuilder builder(unwrap(M));
builder.setEngineKind(EngineKind::JIT)
.setErrorStr(&Error)
+ .setUseMCJIT(true)
.setOptLevel((CodeGenOpt::Level)options.OptLevel)
.setCodeModel(unwrap(options.CodeModel))
.setTargetOptions(targetOptions);
}
void LLVMFreeMachineCodeForFunction(LLVMExecutionEngineRef EE, LLVMValueRef F) {
+ unwrap(EE)->freeMachineCodeForFunction(unwrap<Function>(F));
}
void LLVMAddModule(LLVMExecutionEngineRef EE, LLVMModuleRef M){
void *LLVMRecompileAndRelinkFunction(LLVMExecutionEngineRef EE,
LLVMValueRef Fn) {
- return nullptr;
+ return unwrap(EE)->recompileAndRelinkFunction(unwrap<Function>(Fn));
}
LLVMTargetDataRef LLVMGetExecutionEngineTargetData(LLVMExecutionEngineRef EE) {
return nullptr;
}
+ /// recompileAndRelinkFunction - For the interpreter, functions are always
+ /// up-to-date.
+ ///
+ void *recompileAndRelinkFunction(Function *F) override {
+ return getPointerToFunction(F);
+ }
+
+ /// freeMachineCodeForFunction - The interpreter does not generate any code.
+ ///
+ void freeMachineCodeForFunction(Function *F) override { }
+
// Methods used to execute code:
// Place a call on the stack
void callFunction(Function *F, const std::vector<GenericValue> &ArgVals);
void SwitchToNewBasicBlock(BasicBlock *Dest, ExecutionContext &SF);
void *getPointerToFunction(Function *F) override { return (void*)F; }
+ void *getPointerToBasicBlock(BasicBlock *BB) override { return (void*)BB; }
void initializeExecutionEngine() { }
void initializeExternalFunctions();
--- /dev/null
+# TODO: Support other architectures. See Makefile.
+add_definitions(-DENABLE_X86_JIT)
+
+add_llvm_library(LLVMJIT
+ JIT.cpp
+ JITEmitter.cpp
+ JITMemoryManager.cpp
+ )
--- /dev/null
+//===-- JIT.cpp - LLVM Just in Time Compiler ------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This tool implements a just-in-time compiler for LLVM, allowing direct
+// execution of LLVM bitcode in an efficient manner.
+//
+//===----------------------------------------------------------------------===//
+
+#include "JIT.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/CodeGen/MachineCodeInfo.h"
+#include "llvm/Config/config.h"
+#include "llvm/ExecutionEngine/GenericValue.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
+#include "llvm/ExecutionEngine/JITMemoryManager.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/Dwarf.h"
+#include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/MutexGuard.h"
+#include "llvm/Target/TargetJITInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+
+using namespace llvm;
+
+#ifdef __APPLE__
+// Apple gcc defaults to -fuse-cxa-atexit (i.e. calls __cxa_atexit instead
+// of atexit). It passes the address of linker generated symbol __dso_handle
+// to the function.
+// This configuration change happened at version 5330.
+# include <AvailabilityMacros.h>
+# if defined(MAC_OS_X_VERSION_10_4) && \
+ ((MAC_OS_X_VERSION_MIN_REQUIRED > MAC_OS_X_VERSION_10_4) || \
+ (MAC_OS_X_VERSION_MIN_REQUIRED == MAC_OS_X_VERSION_10_4 && \
+ __APPLE_CC__ >= 5330))
+# ifndef HAVE___DSO_HANDLE
+# define HAVE___DSO_HANDLE 1
+# endif
+# endif
+#endif
+
+#if HAVE___DSO_HANDLE
+extern void *__dso_handle __attribute__ ((__visibility__ ("hidden")));
+#endif
+
+namespace {
+
+static struct RegisterJIT {
+ RegisterJIT() { JIT::Register(); }
+} JITRegistrator;
+
+}
+
+extern "C" void LLVMLinkInJIT() {
+}
+
+/// createJIT - This is the factory method for creating a JIT for the current
+/// machine, it does not fall back to the interpreter. This takes ownership
+/// of the module.
+ExecutionEngine *JIT::createJIT(Module *M,
+ std::string *ErrorStr,
+ JITMemoryManager *JMM,
+ bool GVsWithCode,
+ TargetMachine *TM) {
+ // Try to register the program as a source of symbols to resolve against.
+ //
+ // FIXME: Don't do this here.
+ sys::DynamicLibrary::LoadLibraryPermanently(nullptr, nullptr);
+
+ // If the target supports JIT code generation, create the JIT.
+ if (TargetJITInfo *TJ = TM->getSubtargetImpl()->getJITInfo()) {
+ return new JIT(M, *TM, *TJ, JMM, GVsWithCode);
+ } else {
+ if (ErrorStr)
+ *ErrorStr = "target does not support JIT code generation";
+ return nullptr;
+ }
+}
+
+namespace {
+/// This class supports the global getPointerToNamedFunction(), which allows
+/// bugpoint or gdb users to search for a function by name without any context.
+class JitPool {
+ SmallPtrSet<JIT*, 1> JITs; // Optimize for process containing just 1 JIT.
+ mutable sys::Mutex Lock;
+public:
+ void Add(JIT *jit) {
+ MutexGuard guard(Lock);
+ JITs.insert(jit);
+ }
+ void Remove(JIT *jit) {
+ MutexGuard guard(Lock);
+ JITs.erase(jit);
+ }
+ void *getPointerToNamedFunction(const char *Name) const {
+ MutexGuard guard(Lock);
+ assert(JITs.size() != 0 && "No Jit registered");
+ //search function in every instance of JIT
+ for (SmallPtrSet<JIT*, 1>::const_iterator Jit = JITs.begin(),
+ end = JITs.end();
+ Jit != end; ++Jit) {
+ if (Function *F = (*Jit)->FindFunctionNamed(Name))
+ return (*Jit)->getPointerToFunction(F);
+ }
+ // The function is not available : fallback on the first created (will
+ // search in symbol of the current program/library)
+ return (*JITs.begin())->getPointerToNamedFunction(Name);
+ }
+};
+ManagedStatic<JitPool> AllJits;
+}
+extern "C" {
+ // getPointerToNamedFunction - This function is used as a global wrapper to
+ // JIT::getPointerToNamedFunction for the purpose of resolving symbols when
+ // bugpoint is debugging the JIT. In that scenario, we are loading an .so and
+ // need to resolve function(s) that are being mis-codegenerated, so we need to
+ // resolve their addresses at runtime, and this is the way to do it.
+ void *getPointerToNamedFunction(const char *Name) {
+ return AllJits->getPointerToNamedFunction(Name);
+ }
+}
+
+JIT::JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji,
+ JITMemoryManager *jmm, bool GVsWithCode)
+ : ExecutionEngine(M), TM(tm), TJI(tji),
+ JMM(jmm ? jmm : JITMemoryManager::CreateDefaultMemManager()),
+ AllocateGVsWithCode(GVsWithCode), isAlreadyCodeGenerating(false) {
+ setDataLayout(TM.getSubtargetImpl()->getDataLayout());
+
+ jitstate = new JITState(M);
+
+ // Initialize JCE
+ JCE = createEmitter(*this, JMM, TM);
+
+ // Register in global list of all JITs.
+ AllJits->Add(this);
+
+ // Add target data
+ MutexGuard locked(lock);
+ FunctionPassManager &PM = jitstate->getPM();
+ M->setDataLayout(TM.getSubtargetImpl()->getDataLayout());
+ PM.add(new DataLayoutPass(M));
+
+ // Turn the machine code intermediate representation into bytes in memory that
+ // may be executed.
+ if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) {
+ report_fatal_error("Target does not support machine code emission!");
+ }
+
+ // Initialize passes.
+ PM.doInitialization();
+}
+
+JIT::~JIT() {
+ // Cleanup.
+ AllJits->Remove(this);
+ delete jitstate;
+ delete JCE;
+ // JMM is a ownership of JCE, so we no need delete JMM here.
+ delete &TM;
+}
+
+/// addModule - Add a new Module to the JIT. If we previously removed the last
+/// Module, we need re-initialize jitstate with a valid Module.
+void JIT::addModule(Module *M) {
+ MutexGuard locked(lock);
+
+ if (Modules.empty()) {
+ assert(!jitstate && "jitstate should be NULL if Modules vector is empty!");
+
+ jitstate = new JITState(M);
+
+ FunctionPassManager &PM = jitstate->getPM();
+ M->setDataLayout(TM.getSubtargetImpl()->getDataLayout());
+ PM.add(new DataLayoutPass(M));
+
+ // Turn the machine code intermediate representation into bytes in memory
+ // that may be executed.
+ if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) {
+ report_fatal_error("Target does not support machine code emission!");
+ }
+
+ // Initialize passes.
+ PM.doInitialization();
+ }
+
+ ExecutionEngine::addModule(M);
+}
+
+/// removeModule - If we are removing the last Module, invalidate the jitstate
+/// since the PassManager it contains references a released Module.
+bool JIT::removeModule(Module *M) {
+ bool result = ExecutionEngine::removeModule(M);
+
+ MutexGuard locked(lock);
+
+ if (jitstate && jitstate->getModule() == M) {
+ delete jitstate;
+ jitstate = nullptr;
+ }
+
+ if (!jitstate && !Modules.empty()) {
+ jitstate = new JITState(Modules[0]);
+
+ FunctionPassManager &PM = jitstate->getPM();
+ M->setDataLayout(TM.getSubtargetImpl()->getDataLayout());
+ PM.add(new DataLayoutPass(M));
+
+ // Turn the machine code intermediate representation into bytes in memory
+ // that may be executed.
+ if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) {
+ report_fatal_error("Target does not support machine code emission!");
+ }
+
+ // Initialize passes.
+ PM.doInitialization();
+ }
+ return result;
+}
+
+/// run - Start execution with the specified function and arguments.
+///
+GenericValue JIT::runFunction(Function *F,
+ const std::vector<GenericValue> &ArgValues) {
+ assert(F && "Function *F was null at entry to run()");
+
+ void *FPtr = getPointerToFunction(F);
+ assert(FPtr && "Pointer to fn's code was null after getPointerToFunction");
+ FunctionType *FTy = F->getFunctionType();
+ Type *RetTy = FTy->getReturnType();
+
+ assert((FTy->getNumParams() == ArgValues.size() ||
+ (FTy->isVarArg() && FTy->getNumParams() <= ArgValues.size())) &&
+ "Wrong number of arguments passed into function!");
+ assert(FTy->getNumParams() == ArgValues.size() &&
+ "This doesn't support passing arguments through varargs (yet)!");
+
+ // Handle some common cases first. These cases correspond to common `main'
+ // prototypes.
+ if (RetTy->isIntegerTy(32) || RetTy->isVoidTy()) {
+ switch (ArgValues.size()) {
+ case 3:
+ if (FTy->getParamType(0)->isIntegerTy(32) &&
+ FTy->getParamType(1)->isPointerTy() &&
+ FTy->getParamType(2)->isPointerTy()) {
+ int (*PF)(int, char **, const char **) =
+ (int(*)(int, char **, const char **))(intptr_t)FPtr;
+
+ // Call the function.
+ GenericValue rv;
+ rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(),
+ (char **)GVTOP(ArgValues[1]),
+ (const char **)GVTOP(ArgValues[2])));
+ return rv;
+ }
+ break;
+ case 2:
+ if (FTy->getParamType(0)->isIntegerTy(32) &&
+ FTy->getParamType(1)->isPointerTy()) {
+ int (*PF)(int, char **) = (int(*)(int, char **))(intptr_t)FPtr;
+
+ // Call the function.
+ GenericValue rv;
+ rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(),
+ (char **)GVTOP(ArgValues[1])));
+ return rv;
+ }
+ break;
+ case 1:
+ if (FTy->getParamType(0)->isIntegerTy(32)) {
+ GenericValue rv;
+ int (*PF)(int) = (int(*)(int))(intptr_t)FPtr;
+ rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue()));
+ return rv;
+ }
+ if (FTy->getParamType(0)->isPointerTy()) {
+ GenericValue rv;
+ int (*PF)(char *) = (int(*)(char *))(intptr_t)FPtr;
+ rv.IntVal = APInt(32, PF((char*)GVTOP(ArgValues[0])));
+ return rv;
+ }
+ break;
+ }
+ }
+
+ // Handle cases where no arguments are passed first.
+ if (ArgValues.empty()) {
+ GenericValue rv;
+ switch (RetTy->getTypeID()) {
+ default: llvm_unreachable("Unknown return type for function call!");
+ case Type::IntegerTyID: {
+ unsigned BitWidth = cast<IntegerType>(RetTy)->getBitWidth();
+ if (BitWidth == 1)
+ rv.IntVal = APInt(BitWidth, ((bool(*)())(intptr_t)FPtr)());
+ else if (BitWidth <= 8)
+ rv.IntVal = APInt(BitWidth, ((char(*)())(intptr_t)FPtr)());
+ else if (BitWidth <= 16)
+ rv.IntVal = APInt(BitWidth, ((short(*)())(intptr_t)FPtr)());
+ else if (BitWidth <= 32)
+ rv.IntVal = APInt(BitWidth, ((int(*)())(intptr_t)FPtr)());
+ else if (BitWidth <= 64)
+ rv.IntVal = APInt(BitWidth, ((int64_t(*)())(intptr_t)FPtr)());
+ else
+ llvm_unreachable("Integer types > 64 bits not supported");
+ return rv;
+ }
+ case Type::VoidTyID:
+ rv.IntVal = APInt(32, ((int(*)())(intptr_t)FPtr)());
+ return rv;
+ case Type::FloatTyID:
+ rv.FloatVal = ((float(*)())(intptr_t)FPtr)();
+ return rv;
+ case Type::DoubleTyID:
+ rv.DoubleVal = ((double(*)())(intptr_t)FPtr)();
+ return rv;
+ case Type::X86_FP80TyID:
+ case Type::FP128TyID:
+ case Type::PPC_FP128TyID:
+ llvm_unreachable("long double not supported yet");
+ case Type::PointerTyID:
+ return PTOGV(((void*(*)())(intptr_t)FPtr)());
+ }
+ }
+
+ // Okay, this is not one of our quick and easy cases. Because we don't have a
+ // full FFI, we have to codegen a nullary stub function that just calls the
+ // function we are interested in, passing in constants for all of the
+ // arguments. Make this function and return.
+
+ // First, create the function.
+ FunctionType *STy=FunctionType::get(RetTy, false);
+ Function *Stub = Function::Create(STy, Function::InternalLinkage, "",
+ F->getParent());
+
+ // Insert a basic block.
+ BasicBlock *StubBB = BasicBlock::Create(F->getContext(), "", Stub);
+
+ // Convert all of the GenericValue arguments over to constants. Note that we
+ // currently don't support varargs.
+ SmallVector<Value*, 8> Args;
+ for (unsigned i = 0, e = ArgValues.size(); i != e; ++i) {
+ Constant *C = nullptr;
+ Type *ArgTy = FTy->getParamType(i);
+ const GenericValue &AV = ArgValues[i];
+ switch (ArgTy->getTypeID()) {
+ default: llvm_unreachable("Unknown argument type for function call!");
+ case Type::IntegerTyID:
+ C = ConstantInt::get(F->getContext(), AV.IntVal);
+ break;
+ case Type::FloatTyID:
+ C = ConstantFP::get(F->getContext(), APFloat(AV.FloatVal));
+ break;
+ case Type::DoubleTyID:
+ C = ConstantFP::get(F->getContext(), APFloat(AV.DoubleVal));
+ break;
+ case Type::PPC_FP128TyID:
+ case Type::X86_FP80TyID:
+ case Type::FP128TyID:
+ C = ConstantFP::get(F->getContext(), APFloat(ArgTy->getFltSemantics(),
+ AV.IntVal));
+ break;
+ case Type::PointerTyID:
+ void *ArgPtr = GVTOP(AV);
+ if (sizeof(void*) == 4)
+ C = ConstantInt::get(Type::getInt32Ty(F->getContext()),
+ (int)(intptr_t)ArgPtr);
+ else
+ C = ConstantInt::get(Type::getInt64Ty(F->getContext()),
+ (intptr_t)ArgPtr);
+ // Cast the integer to pointer
+ C = ConstantExpr::getIntToPtr(C, ArgTy);
+ break;
+ }
+ Args.push_back(C);
+ }
+
+ CallInst *TheCall = CallInst::Create(F, Args, "", StubBB);
+ TheCall->setCallingConv(F->getCallingConv());
+ TheCall->setTailCall();
+ if (!TheCall->getType()->isVoidTy())
+ // Return result of the call.
+ ReturnInst::Create(F->getContext(), TheCall, StubBB);
+ else
+ ReturnInst::Create(F->getContext(), StubBB); // Just return void.
+
+ // Finally, call our nullary stub function.
+ GenericValue Result = runFunction(Stub, std::vector<GenericValue>());
+ // Erase it, since no other function can have a reference to it.
+ Stub->eraseFromParent();
+ // And return the result.
+ return Result;
+}
+
+void JIT::RegisterJITEventListener(JITEventListener *L) {
+ if (!L)
+ return;
+ MutexGuard locked(lock);
+ EventListeners.push_back(L);
+}
+void JIT::UnregisterJITEventListener(JITEventListener *L) {
+ if (!L)
+ return;
+ MutexGuard locked(lock);
+ std::vector<JITEventListener*>::reverse_iterator I=
+ std::find(EventListeners.rbegin(), EventListeners.rend(), L);
+ if (I != EventListeners.rend()) {
+ std::swap(*I, EventListeners.back());
+ EventListeners.pop_back();
+ }
+}
+void JIT::NotifyFunctionEmitted(
+ const Function &F,
+ void *Code, size_t Size,
+ const JITEvent_EmittedFunctionDetails &Details) {
+ MutexGuard locked(lock);
+ for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
+ EventListeners[I]->NotifyFunctionEmitted(F, Code, Size, Details);
+ }
+}
+
+void JIT::NotifyFreeingMachineCode(void *OldPtr) {
+ MutexGuard locked(lock);
+ for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
+ EventListeners[I]->NotifyFreeingMachineCode(OldPtr);
+ }
+}
+
+/// runJITOnFunction - Run the FunctionPassManager full of
+/// just-in-time compilation passes on F, hopefully filling in
+/// GlobalAddress[F] with the address of F's machine code.
+///
+void JIT::runJITOnFunction(Function *F, MachineCodeInfo *MCI) {
+ MutexGuard locked(lock);
+
+ class MCIListener : public JITEventListener {
+ MachineCodeInfo *const MCI;
+ public:
+ MCIListener(MachineCodeInfo *mci) : MCI(mci) {}
+ void NotifyFunctionEmitted(const Function &, void *Code, size_t Size,
+ const EmittedFunctionDetails &) override {
+ MCI->setAddress(Code);
+ MCI->setSize(Size);
+ }
+ };
+ MCIListener MCIL(MCI);
+ if (MCI)
+ RegisterJITEventListener(&MCIL);
+
+ runJITOnFunctionUnlocked(F);
+
+ if (MCI)
+ UnregisterJITEventListener(&MCIL);
+}
+
+void JIT::runJITOnFunctionUnlocked(Function *F) {
+ assert(!isAlreadyCodeGenerating && "Error: Recursive compilation detected!");
+
+ jitTheFunctionUnlocked(F);
+
+ // If the function referred to another function that had not yet been
+ // read from bitcode, and we are jitting non-lazily, emit it now.
+ while (!jitstate->getPendingFunctions().empty()) {
+ Function *PF = jitstate->getPendingFunctions().back();
+ jitstate->getPendingFunctions().pop_back();
+
+ assert(!PF->hasAvailableExternallyLinkage() &&
+ "Externally-defined function should not be in pending list.");
+
+ jitTheFunctionUnlocked(PF);
+
+ // Now that the function has been jitted, ask the JITEmitter to rewrite
+ // the stub with real address of the function.
+ updateFunctionStubUnlocked(PF);
+ }
+}
+
+void JIT::jitTheFunctionUnlocked(Function *F) {
+ isAlreadyCodeGenerating = true;
+ jitstate->getPM().run(*F);
+ isAlreadyCodeGenerating = false;
+
+ // clear basic block addresses after this function is done
+ getBasicBlockAddressMap().clear();
+}
+
+/// getPointerToFunction - This method is used to get the address of the
+/// specified function, compiling it if necessary.
+///
+void *JIT::getPointerToFunction(Function *F) {
+
+ if (void *Addr = getPointerToGlobalIfAvailable(F))
+ return Addr; // Check if function already code gen'd
+
+ MutexGuard locked(lock);
+
+ // Now that this thread owns the lock, make sure we read in the function if it
+ // exists in this Module.
+ std::string ErrorMsg;
+ if (F->Materialize(&ErrorMsg)) {
+ report_fatal_error("Error reading function '" + F->getName()+
+ "' from bitcode file: " + ErrorMsg);
+ }
+
+ // ... and check if another thread has already code gen'd the function.
+ if (void *Addr = getPointerToGlobalIfAvailable(F))
+ return Addr;
+
+ if (F->isDeclaration() || F->hasAvailableExternallyLinkage()) {
+ bool AbortOnFailure = !F->hasExternalWeakLinkage();
+ void *Addr = getPointerToNamedFunction(F->getName(), AbortOnFailure);
+ addGlobalMapping(F, Addr);
+ return Addr;
+ }
+
+ runJITOnFunctionUnlocked(F);
+
+ void *Addr = getPointerToGlobalIfAvailable(F);
+ assert(Addr && "Code generation didn't add function to GlobalAddress table!");
+ return Addr;
+}
+
+void JIT::addPointerToBasicBlock(const BasicBlock *BB, void *Addr) {
+ MutexGuard locked(lock);
+
+ BasicBlockAddressMapTy::iterator I =
+ getBasicBlockAddressMap().find(BB);
+ if (I == getBasicBlockAddressMap().end()) {
+ getBasicBlockAddressMap()[BB] = Addr;
+ } else {
+ // ignore repeats: some BBs can be split into few MBBs?
+ }
+}
+
+void JIT::clearPointerToBasicBlock(const BasicBlock *BB) {
+ MutexGuard locked(lock);
+ getBasicBlockAddressMap().erase(BB);
+}
+
+void *JIT::getPointerToBasicBlock(BasicBlock *BB) {
+ // make sure it's function is compiled by JIT
+ (void)getPointerToFunction(BB->getParent());
+
+ // resolve basic block address
+ MutexGuard locked(lock);
+
+ BasicBlockAddressMapTy::iterator I =
+ getBasicBlockAddressMap().find(BB);
+ if (I != getBasicBlockAddressMap().end()) {
+ return I->second;
+ } else {
+ llvm_unreachable("JIT does not have BB address for address-of-label, was"
+ " it eliminated by optimizer?");
+ }
+}
+
+void *JIT::getPointerToNamedFunction(const std::string &Name,
+ bool AbortOnFailure){
+ if (!isSymbolSearchingDisabled()) {
+ void *ptr = JMM->getPointerToNamedFunction(Name, false);
+ if (ptr)
+ return ptr;
+ }
+
+ /// If a LazyFunctionCreator is installed, use it to get/create the function.
+ if (LazyFunctionCreator)
+ if (void *RP = LazyFunctionCreator(Name))
+ return RP;
+
+ if (AbortOnFailure) {
+ report_fatal_error("Program used external function '"+Name+
+ "' which could not be resolved!");
+ }
+ return nullptr;
+}
+
+
+/// getOrEmitGlobalVariable - Return the address of the specified global
+/// variable, possibly emitting it to memory if needed. This is used by the
+/// Emitter.
+void *JIT::getOrEmitGlobalVariable(const GlobalVariable *GV) {
+ MutexGuard locked(lock);
+
+ void *Ptr = getPointerToGlobalIfAvailable(GV);
+ if (Ptr) return Ptr;
+
+ // If the global is external, just remember the address.
+ if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) {
+#if HAVE___DSO_HANDLE
+ if (GV->getName() == "__dso_handle")
+ return (void*)&__dso_handle;
+#endif
+ Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(GV->getName());
+ if (!Ptr) {
+ report_fatal_error("Could not resolve external global address: "
+ +GV->getName());
+ }
+ addGlobalMapping(GV, Ptr);
+ } else {
+ // If the global hasn't been emitted to memory yet, allocate space and
+ // emit it into memory.
+ Ptr = getMemoryForGV(GV);
+ addGlobalMapping(GV, Ptr);
+ EmitGlobalVariable(GV); // Initialize the variable.
+ }
+ return Ptr;
+}
+
+/// recompileAndRelinkFunction - This method is used to force a function
+/// which has already been compiled, to be compiled again, possibly
+/// after it has been modified. Then the entry to the old copy is overwritten
+/// with a branch to the new copy. If there was no old copy, this acts
+/// just like JIT::getPointerToFunction().
+///
+void *JIT::recompileAndRelinkFunction(Function *F) {
+ void *OldAddr = getPointerToGlobalIfAvailable(F);
+
+ // If it's not already compiled there is no reason to patch it up.
+ if (!OldAddr) return getPointerToFunction(F);
+
+ // Delete the old function mapping.
+ addGlobalMapping(F, nullptr);
+
+ // Recodegen the function
+ runJITOnFunction(F);
+
+ // Update state, forward the old function to the new function.
+ void *Addr = getPointerToGlobalIfAvailable(F);
+ assert(Addr && "Code generation didn't add function to GlobalAddress table!");
+ TJI.replaceMachineCodeForFunction(OldAddr, Addr);
+ return Addr;
+}
+
+/// getMemoryForGV - This method abstracts memory allocation of global
+/// variable so that the JIT can allocate thread local variables depending
+/// on the target.
+///
+char* JIT::getMemoryForGV(const GlobalVariable* GV) {
+ char *Ptr;
+
+ // GlobalVariable's which are not "constant" will cause trouble in a server
+ // situation. It's returned in the same block of memory as code which may
+ // not be writable.
+ if (isGVCompilationDisabled() && !GV->isConstant()) {
+ report_fatal_error("Compilation of non-internal GlobalValue is disabled!");
+ }
+
+ // Some applications require globals and code to live together, so they may
+ // be allocated into the same buffer, but in general globals are allocated
+ // through the memory manager which puts them near the code but not in the
+ // same buffer.
+ Type *GlobalType = GV->getType()->getElementType();
+ size_t S = getDataLayout()->getTypeAllocSize(GlobalType);
+ size_t A = getDataLayout()->getPreferredAlignment(GV);
+ if (GV->isThreadLocal()) {
+ MutexGuard locked(lock);
+ Ptr = TJI.allocateThreadLocalMemory(S);
+ } else if (TJI.allocateSeparateGVMemory()) {
+ if (A <= 8) {
+ Ptr = (char*)malloc(S);
+ } else {
+ // Allocate S+A bytes of memory, then use an aligned pointer within that
+ // space.
+ Ptr = (char*)malloc(S+A);
+ unsigned MisAligned = ((intptr_t)Ptr & (A-1));
+ Ptr = Ptr + (MisAligned ? (A-MisAligned) : 0);
+ }
+ } else if (AllocateGVsWithCode) {
+ Ptr = (char*)JCE->allocateSpace(S, A);
+ } else {
+ Ptr = (char*)JCE->allocateGlobal(S, A);
+ }
+ return Ptr;
+}
+
+void JIT::addPendingFunction(Function *F) {
+ MutexGuard locked(lock);
+ jitstate->getPendingFunctions().push_back(F);
+}
+
+
+JITEventListener::~JITEventListener() {}
--- /dev/null
+//===-- JIT.h - Class definition for the JIT --------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the top-level JIT data structure.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef JIT_H
+#define JIT_H
+
+#include "llvm/ExecutionEngine/ExecutionEngine.h"
+#include "llvm/IR/ValueHandle.h"
+#include "llvm/PassManager.h"
+
+namespace llvm {
+
+class Function;
+struct JITEvent_EmittedFunctionDetails;
+class MachineCodeEmitter;
+class MachineCodeInfo;
+class TargetJITInfo;
+class TargetMachine;
+
+class JITState {
+private:
+ FunctionPassManager PM; // Passes to compile a function
+ Module *M; // Module used to create the PM
+
+ /// PendingFunctions - Functions which have not been code generated yet, but
+ /// were called from a function being code generated.
+ std::vector<AssertingVH<Function> > PendingFunctions;
+
+public:
+ explicit JITState(Module *M) : PM(M), M(M) {}
+
+ FunctionPassManager &getPM() {
+ return PM;
+ }
+
+ Module *getModule() const { return M; }
+ std::vector<AssertingVH<Function> > &getPendingFunctions() {
+ return PendingFunctions;
+ }
+};
+
+
+class JIT : public ExecutionEngine {
+ /// types
+ typedef ValueMap<const BasicBlock *, void *>
+ BasicBlockAddressMapTy;
+ /// data
+ TargetMachine &TM; // The current target we are compiling to
+ TargetJITInfo &TJI; // The JITInfo for the target we are compiling to
+ JITCodeEmitter *JCE; // JCE object
+ JITMemoryManager *JMM;
+ std::vector<JITEventListener*> EventListeners;
+
+ /// AllocateGVsWithCode - Some applications require that global variables and
+ /// code be allocated into the same region of memory, in which case this flag
+ /// should be set to true. Doing so breaks freeMachineCodeForFunction.
+ bool AllocateGVsWithCode;
+
+ /// True while the JIT is generating code. Used to assert against recursive
+ /// entry.
+ bool isAlreadyCodeGenerating;
+
+ JITState *jitstate;
+
+ /// BasicBlockAddressMap - A mapping between LLVM basic blocks and their
+ /// actualized version, only filled for basic blocks that have their address
+ /// taken.
+ BasicBlockAddressMapTy BasicBlockAddressMap;
+
+
+ JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji,
+ JITMemoryManager *JMM, bool AllocateGVsWithCode);
+public:
+ ~JIT();
+
+ static void Register() {
+ JITCtor = createJIT;
+ }
+
+ /// getJITInfo - Return the target JIT information structure.
+ ///
+ TargetJITInfo &getJITInfo() const { return TJI; }
+
+ void addModule(Module *M) override;
+
+ /// removeModule - Remove a Module from the list of modules. Returns true if
+ /// M is found.
+ bool removeModule(Module *M) override;
+
+ /// runFunction - Start execution with the specified function and arguments.
+ ///
+ GenericValue runFunction(Function *F,
+ const std::vector<GenericValue> &ArgValues) override;
+
+ /// getPointerToNamedFunction - This method returns the address of the
+ /// specified function by using the MemoryManager. As such it is only
+ /// useful for resolving library symbols, not code generated symbols.
+ ///
+ /// If AbortOnFailure is false and no function with the given name is
+ /// found, this function silently returns a null pointer. Otherwise,
+ /// it prints a message to stderr and aborts.
+ ///
+ void *getPointerToNamedFunction(const std::string &Name,
+ bool AbortOnFailure = true) override;
+
+ // CompilationCallback - Invoked the first time that a call site is found,
+ // which causes lazy compilation of the target function.
+ //
+ static void CompilationCallback();
+
+ /// getPointerToFunction - This returns the address of the specified function,
+ /// compiling it if necessary.
+ ///
+ void *getPointerToFunction(Function *F) override;
+
+ /// addPointerToBasicBlock - Adds address of the specific basic block.
+ void addPointerToBasicBlock(const BasicBlock *BB, void *Addr);
+
+ /// clearPointerToBasicBlock - Removes address of specific basic block.
+ void clearPointerToBasicBlock(const BasicBlock *BB);
+
+ /// getPointerToBasicBlock - This returns the address of the specified basic
+ /// block, assuming function is compiled.
+ void *getPointerToBasicBlock(BasicBlock *BB) override;
+
+ /// getOrEmitGlobalVariable - Return the address of the specified global
+ /// variable, possibly emitting it to memory if needed. This is used by the
+ /// Emitter.
+ void *getOrEmitGlobalVariable(const GlobalVariable *GV) override;
+
+ /// getPointerToFunctionOrStub - If the specified function has been
+ /// code-gen'd, return a pointer to the function. If not, compile it, or use
+ /// a stub to implement lazy compilation if available.
+ ///
+ void *getPointerToFunctionOrStub(Function *F) override;
+
+ /// recompileAndRelinkFunction - This method is used to force a function
+ /// which has already been compiled, to be compiled again, possibly
+ /// after it has been modified. Then the entry to the old copy is overwritten
+ /// with a branch to the new copy. If there was no old copy, this acts
+ /// just like JIT::getPointerToFunction().
+ ///
+ void *recompileAndRelinkFunction(Function *F) override;
+
+ /// freeMachineCodeForFunction - deallocate memory used to code-generate this
+ /// Function.
+ ///
+ void freeMachineCodeForFunction(Function *F) override;
+
+ /// addPendingFunction - while jitting non-lazily, a called but non-codegen'd
+ /// function was encountered. Add it to a pending list to be processed after
+ /// the current function.
+ ///
+ void addPendingFunction(Function *F);
+
+ /// getCodeEmitter - Return the code emitter this JIT is emitting into.
+ ///
+ JITCodeEmitter *getCodeEmitter() const { return JCE; }
+
+ static ExecutionEngine *createJIT(Module *M,
+ std::string *ErrorStr,
+ JITMemoryManager *JMM,
+ bool GVsWithCode,
+ TargetMachine *TM);
+
+ // Run the JIT on F and return information about the generated code
+ void runJITOnFunction(Function *F, MachineCodeInfo *MCI = nullptr) override;
+
+ void RegisterJITEventListener(JITEventListener *L) override;
+ void UnregisterJITEventListener(JITEventListener *L) override;
+
+ TargetMachine *getTargetMachine() override { return &TM; }
+
+ /// These functions correspond to the methods on JITEventListener. They
+ /// iterate over the registered listeners and call the corresponding method on
+ /// each.
+ void NotifyFunctionEmitted(
+ const Function &F, void *Code, size_t Size,
+ const JITEvent_EmittedFunctionDetails &Details);
+ void NotifyFreeingMachineCode(void *OldPtr);
+
+ BasicBlockAddressMapTy &
+ getBasicBlockAddressMap() {
+ return BasicBlockAddressMap;
+ }
+
+
+private:
+ static JITCodeEmitter *createEmitter(JIT &J, JITMemoryManager *JMM,
+ TargetMachine &tm);
+ void runJITOnFunctionUnlocked(Function *F);
+ void updateFunctionStubUnlocked(Function *F);
+ void jitTheFunctionUnlocked(Function *F);
+
+protected:
+
+ /// getMemoryforGV - Allocate memory for a global variable.
+ char* getMemoryForGV(const GlobalVariable* GV) override;
+
+};
+
+} // End llvm namespace
+
+#endif
--- /dev/null
+//===-- JITEmitter.cpp - Write machine code to executable memory ----------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a MachineCodeEmitter object that is used by the JIT to
+// write machine code to memory and remember where relocatable values are.
+//
+//===----------------------------------------------------------------------===//
+
+#include "JIT.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/CodeGen/MachineCodeInfo.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineRelocation.h"
+#include "llvm/ExecutionEngine/GenericValue.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
+#include "llvm/ExecutionEngine/JITMemoryManager.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/ValueHandle.h"
+#include "llvm/IR/ValueMap.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/Memory.h"
+#include "llvm/Support/MutexGuard.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetJITInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
+#include <algorithm>
+#ifndef NDEBUG
+#include <iomanip>
+#endif
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+STATISTIC(NumBytes, "Number of bytes of machine code compiled");
+STATISTIC(NumRelos, "Number of relocations applied");
+STATISTIC(NumRetries, "Number of retries with more memory");
+
+
+// A declaration may stop being a declaration once it's fully read from bitcode.
+// This function returns true if F is fully read and is still a declaration.
+static bool isNonGhostDeclaration(const Function *F) {
+ return F->isDeclaration() && !F->isMaterializable();
+}
+
+//===----------------------------------------------------------------------===//
+// JIT lazy compilation code.
+//
+namespace {
+ class JITEmitter;
+ class JITResolverState;
+
+ template<typename ValueTy>
+ struct NoRAUWValueMapConfig : public ValueMapConfig<ValueTy> {
+ typedef JITResolverState *ExtraData;
+ static void onRAUW(JITResolverState *, Value *Old, Value *New) {
+ llvm_unreachable("The JIT doesn't know how to handle a"
+ " RAUW on a value it has emitted.");
+ }
+ };
+
+ struct CallSiteValueMapConfig : public NoRAUWValueMapConfig<Function*> {
+ typedef JITResolverState *ExtraData;
+ static void onDelete(JITResolverState *JRS, Function *F);
+ };
+
+ class JITResolverState {
+ public:
+ typedef ValueMap<Function*, void*, NoRAUWValueMapConfig<Function*> >
+ FunctionToLazyStubMapTy;
+ typedef std::map<void*, AssertingVH<Function> > CallSiteToFunctionMapTy;
+ typedef ValueMap<Function *, SmallPtrSet<void*, 1>,
+ CallSiteValueMapConfig> FunctionToCallSitesMapTy;
+ typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy;
+ private:
+ /// FunctionToLazyStubMap - Keep track of the lazy stub created for a
+ /// particular function so that we can reuse them if necessary.
+ FunctionToLazyStubMapTy FunctionToLazyStubMap;
+
+ /// CallSiteToFunctionMap - Keep track of the function that each lazy call
+ /// site corresponds to, and vice versa.
+ CallSiteToFunctionMapTy CallSiteToFunctionMap;
+ FunctionToCallSitesMapTy FunctionToCallSitesMap;
+
+ /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a
+ /// particular GlobalVariable so that we can reuse them if necessary.
+ GlobalToIndirectSymMapTy GlobalToIndirectSymMap;
+
+#ifndef NDEBUG
+ /// Instance of the JIT this ResolverState serves.
+ JIT *TheJIT;
+#endif
+
+ public:
+ JITResolverState(JIT *jit) : FunctionToLazyStubMap(this),
+ FunctionToCallSitesMap(this) {
+#ifndef NDEBUG
+ TheJIT = jit;
+#endif
+ }
+
+ FunctionToLazyStubMapTy& getFunctionToLazyStubMap() {
+ return FunctionToLazyStubMap;
+ }
+
+ GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap() {
+ return GlobalToIndirectSymMap;
+ }
+
+ std::pair<void *, Function *> LookupFunctionFromCallSite(
+ void *CallSite) const {
+ // The address given to us for the stub may not be exactly right, it
+ // might be a little bit after the stub. As such, use upper_bound to
+ // find it.
+ CallSiteToFunctionMapTy::const_iterator I =
+ CallSiteToFunctionMap.upper_bound(CallSite);
+ assert(I != CallSiteToFunctionMap.begin() &&
+ "This is not a known call site!");
+ --I;
+ return *I;
+ }
+
+ void AddCallSite(void *CallSite, Function *F) {
+ bool Inserted = CallSiteToFunctionMap.insert(
+ std::make_pair(CallSite, F)).second;
+ (void)Inserted;
+ assert(Inserted && "Pair was already in CallSiteToFunctionMap");
+ FunctionToCallSitesMap[F].insert(CallSite);
+ }
+
+ void EraseAllCallSitesForPrelocked(Function *F);
+
+ // Erases _all_ call sites regardless of their function. This is used to
+ // unregister the stub addresses from the StubToResolverMap in
+ // ~JITResolver().
+ void EraseAllCallSitesPrelocked();
+ };
+
+ /// JITResolver - Keep track of, and resolve, call sites for functions that
+ /// have not yet been compiled.
+ class JITResolver {
+ typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy;
+ typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy;
+ typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy;
+
+ /// LazyResolverFn - The target lazy resolver function that we actually
+ /// rewrite instructions to use.
+ TargetJITInfo::LazyResolverFn LazyResolverFn;
+
+ JITResolverState state;
+
+ /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap
+ /// for external functions. TODO: Of course, external functions don't need
+ /// a lazy stub. It's actually here to make it more likely that far calls
+ /// succeed, but no single stub can guarantee that. I'll remove this in a
+ /// subsequent checkin when I actually fix far calls.
+ std::map<void*, void*> ExternalFnToStubMap;
+
+ /// revGOTMap - map addresses to indexes in the GOT
+ std::map<void*, unsigned> revGOTMap;
+ unsigned nextGOTIndex;
+
+ JITEmitter &JE;
+
+ /// Instance of JIT corresponding to this Resolver.
+ JIT *TheJIT;
+
+ public:
+ explicit JITResolver(JIT &jit, JITEmitter &je)
+ : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) {
+ LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn);
+ }
+
+ ~JITResolver();
+
+ /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's
+ /// lazy-compilation stub if it has already been created.
+ void *getLazyFunctionStubIfAvailable(Function *F);
+
+ /// getLazyFunctionStub - This returns a pointer to a function's
+ /// lazy-compilation stub, creating one on demand as needed.
+ void *getLazyFunctionStub(Function *F);
+
+ /// getExternalFunctionStub - Return a stub for the function at the
+ /// specified address, created lazily on demand.
+ void *getExternalFunctionStub(void *FnAddr);
+
+ /// getGlobalValueIndirectSym - Return an indirect symbol containing the
+ /// specified GV address.
+ void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress);
+
+ /// getGOTIndexForAddress - Return a new or existing index in the GOT for
+ /// an address. This function only manages slots, it does not manage the
+ /// contents of the slots or the memory associated with the GOT.
+ unsigned getGOTIndexForAddr(void *addr);
+
+ /// JITCompilerFn - This function is called to resolve a stub to a compiled
+ /// address. If the LLVM Function corresponding to the stub has not yet
+ /// been compiled, this function compiles it first.
+ static void *JITCompilerFn(void *Stub);
+ };
+
+ class StubToResolverMapTy {
+ /// Map a stub address to a specific instance of a JITResolver so that
+ /// lazily-compiled functions can find the right resolver to use.
+ ///
+ /// Guarded by Lock.
+ std::map<void*, JITResolver*> Map;
+
+ /// Guards Map from concurrent accesses.
+ mutable sys::Mutex Lock;
+
+ public:
+ /// Registers a Stub to be resolved by Resolver.
+ void RegisterStubResolver(void *Stub, JITResolver *Resolver) {
+ MutexGuard guard(Lock);
+ Map.insert(std::make_pair(Stub, Resolver));
+ }
+ /// Unregisters the Stub when it's invalidated.
+ void UnregisterStubResolver(void *Stub) {
+ MutexGuard guard(Lock);
+ Map.erase(Stub);
+ }
+ /// Returns the JITResolver instance that owns the Stub.
+ JITResolver *getResolverFromStub(void *Stub) const {
+ MutexGuard guard(Lock);
+ // The address given to us for the stub may not be exactly right, it might
+ // be a little bit after the stub. As such, use upper_bound to find it.
+ // This is the same trick as in LookupFunctionFromCallSite from
+ // JITResolverState.
+ std::map<void*, JITResolver*>::const_iterator I = Map.upper_bound(Stub);
+ assert(I != Map.begin() && "This is not a known stub!");
+ --I;
+ return I->second;
+ }
+ /// True if any stubs refer to the given resolver. Only used in an assert().
+ /// O(N)
+ bool ResolverHasStubs(JITResolver* Resolver) const {
+ MutexGuard guard(Lock);
+ for (std::map<void*, JITResolver*>::const_iterator I = Map.begin(),
+ E = Map.end(); I != E; ++I) {
+ if (I->second == Resolver)
+ return true;
+ }
+ return false;
+ }
+ };
+ /// This needs to be static so that a lazy call stub can access it with no
+ /// context except the address of the stub.
+ ManagedStatic<StubToResolverMapTy> StubToResolverMap;
+
+ /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is
+ /// used to output functions to memory for execution.
+ class JITEmitter : public JITCodeEmitter {
+ JITMemoryManager *MemMgr;
+
+ // When outputting a function stub in the context of some other function, we
+ // save BufferBegin/BufferEnd/CurBufferPtr here.
+ uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr;
+
+ // When reattempting to JIT a function after running out of space, we store
+ // the estimated size of the function we're trying to JIT here, so we can
+ // ask the memory manager for at least this much space. When we
+ // successfully emit the function, we reset this back to zero.
+ uintptr_t SizeEstimate;
+
+ /// Relocations - These are the relocations that the function needs, as
+ /// emitted.
+ std::vector<MachineRelocation> Relocations;
+
+ /// MBBLocations - This vector is a mapping from MBB ID's to their address.
+ /// It is filled in by the StartMachineBasicBlock callback and queried by
+ /// the getMachineBasicBlockAddress callback.
+ std::vector<uintptr_t> MBBLocations;
+
+ /// ConstantPool - The constant pool for the current function.
+ ///
+ MachineConstantPool *ConstantPool;
+
+ /// ConstantPoolBase - A pointer to the first entry in the constant pool.
+ ///
+ void *ConstantPoolBase;
+
+ /// ConstPoolAddresses - Addresses of individual constant pool entries.
+ ///
+ SmallVector<uintptr_t, 8> ConstPoolAddresses;
+
+ /// JumpTable - The jump tables for the current function.
+ ///
+ MachineJumpTableInfo *JumpTable;
+
+ /// JumpTableBase - A pointer to the first entry in the jump table.
+ ///
+ void *JumpTableBase;
+
+ /// Resolver - This contains info about the currently resolved functions.
+ JITResolver Resolver;
+
+ /// LabelLocations - This vector is a mapping from Label ID's to their
+ /// address.
+ DenseMap<MCSymbol*, uintptr_t> LabelLocations;
+
+ /// MMI - Machine module info for exception informations
+ MachineModuleInfo* MMI;
+
+ // CurFn - The llvm function being emitted. Only valid during
+ // finishFunction().
+ const Function *CurFn;
+
+ /// Information about emitted code, which is passed to the
+ /// JITEventListeners. This is reset in startFunction and used in
+ /// finishFunction.
+ JITEvent_EmittedFunctionDetails EmissionDetails;
+
+ struct EmittedCode {
+ void *FunctionBody; // Beginning of the function's allocation.
+ void *Code; // The address the function's code actually starts at.
+ void *ExceptionTable;
+ EmittedCode() : FunctionBody(nullptr), Code(nullptr),
+ ExceptionTable(nullptr) {}
+ };
+ struct EmittedFunctionConfig : public ValueMapConfig<const Function*> {
+ typedef JITEmitter *ExtraData;
+ static void onDelete(JITEmitter *, const Function*);
+ static void onRAUW(JITEmitter *, const Function*, const Function*);
+ };
+ ValueMap<const Function *, EmittedCode,
+ EmittedFunctionConfig> EmittedFunctions;
+
+ DebugLoc PrevDL;
+
+ /// Instance of the JIT
+ JIT *TheJIT;
+
+ public:
+ JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM)
+ : SizeEstimate(0), Resolver(jit, *this), MMI(nullptr), CurFn(nullptr),
+ EmittedFunctions(this), TheJIT(&jit) {
+ MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager();
+ if (jit.getJITInfo().needsGOT()) {
+ MemMgr->AllocateGOT();
+ DEBUG(dbgs() << "JIT is managing a GOT\n");
+ }
+
+ }
+ ~JITEmitter() {
+ delete MemMgr;
+ }
+
+ JITResolver &getJITResolver() { return Resolver; }
+
+ void startFunction(MachineFunction &F) override;
+ bool finishFunction(MachineFunction &F) override;
+
+ void emitConstantPool(MachineConstantPool *MCP);
+ void initJumpTableInfo(MachineJumpTableInfo *MJTI);
+ void emitJumpTableInfo(MachineJumpTableInfo *MJTI);
+
+ void startGVStub(const GlobalValue* GV,
+ unsigned StubSize, unsigned Alignment = 1);
+ void startGVStub(void *Buffer, unsigned StubSize);
+ void finishGVStub();
+ void *allocIndirectGV(const GlobalValue *GV, const uint8_t *Buffer,
+ size_t Size, unsigned Alignment) override;
+
+ /// allocateSpace - Reserves space in the current block if any, or
+ /// allocate a new one of the given size.
+ void *allocateSpace(uintptr_t Size, unsigned Alignment) override;
+
+ /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace,
+ /// this method does not allocate memory in the current output buffer,
+ /// because a global may live longer than the current function.
+ void *allocateGlobal(uintptr_t Size, unsigned Alignment) override;
+
+ void addRelocation(const MachineRelocation &MR) override {
+ Relocations.push_back(MR);
+ }
+
+ void StartMachineBasicBlock(MachineBasicBlock *MBB) override {
+ if (MBBLocations.size() <= (unsigned)MBB->getNumber())
+ MBBLocations.resize((MBB->getNumber()+1)*2);
+ MBBLocations[MBB->getNumber()] = getCurrentPCValue();
+ if (MBB->hasAddressTaken())
+ TheJIT->addPointerToBasicBlock(MBB->getBasicBlock(),
+ (void*)getCurrentPCValue());
+ DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at ["
+ << (void*) getCurrentPCValue() << "]\n");
+ }
+
+ uintptr_t getConstantPoolEntryAddress(unsigned Entry) const override;
+ uintptr_t getJumpTableEntryAddress(unsigned Entry) const override;
+
+ uintptr_t
+ getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override {
+ assert(MBBLocations.size() > (unsigned)MBB->getNumber() &&
+ MBBLocations[MBB->getNumber()] && "MBB not emitted!");
+ return MBBLocations[MBB->getNumber()];
+ }
+
+ /// retryWithMoreMemory - Log a retry and deallocate all memory for the
+ /// given function. Increase the minimum allocation size so that we get
+ /// more memory next time.
+ void retryWithMoreMemory(MachineFunction &F);
+
+ /// deallocateMemForFunction - Deallocate all memory for the specified
+ /// function body.
+ void deallocateMemForFunction(const Function *F);
+
+ void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) override;
+
+ void emitLabel(MCSymbol *Label) override {
+ LabelLocations[Label] = getCurrentPCValue();
+ }
+
+ DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() override {
+ return &LabelLocations;
+ }
+
+ uintptr_t getLabelAddress(MCSymbol *Label) const override {
+ assert(LabelLocations.count(Label) && "Label not emitted!");
+ return LabelLocations.find(Label)->second;
+ }
+
+ void setModuleInfo(MachineModuleInfo* Info) override {
+ MMI = Info;
+ }
+
+ private:
+ void *getPointerToGlobal(GlobalValue *GV, void *Reference,
+ bool MayNeedFarStub);
+ void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference);
+ };
+}
+
+void CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) {
+ JRS->EraseAllCallSitesForPrelocked(F);
+}
+
+void JITResolverState::EraseAllCallSitesForPrelocked(Function *F) {
+ FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F);
+ if (F2C == FunctionToCallSitesMap.end())
+ return;
+ StubToResolverMapTy &S2RMap = *StubToResolverMap;
+ for (SmallPtrSet<void*, 1>::const_iterator I = F2C->second.begin(),
+ E = F2C->second.end(); I != E; ++I) {
+ S2RMap.UnregisterStubResolver(*I);
+ bool Erased = CallSiteToFunctionMap.erase(*I);
+ (void)Erased;
+ assert(Erased && "Missing call site->function mapping");
+ }
+ FunctionToCallSitesMap.erase(F2C);
+}
+
+void JITResolverState::EraseAllCallSitesPrelocked() {
+ StubToResolverMapTy &S2RMap = *StubToResolverMap;
+ for (CallSiteToFunctionMapTy::const_iterator
+ I = CallSiteToFunctionMap.begin(),
+ E = CallSiteToFunctionMap.end(); I != E; ++I) {
+ S2RMap.UnregisterStubResolver(I->first);
+ }
+ CallSiteToFunctionMap.clear();
+ FunctionToCallSitesMap.clear();
+}
+
+JITResolver::~JITResolver() {
+ // No need to lock because we're in the destructor, and state isn't shared.
+ state.EraseAllCallSitesPrelocked();
+ assert(!StubToResolverMap->ResolverHasStubs(this) &&
+ "Resolver destroyed with stubs still alive.");
+}
+
+/// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub
+/// if it has already been created.
+void *JITResolver::getLazyFunctionStubIfAvailable(Function *F) {
+ MutexGuard locked(TheJIT->lock);
+
+ // If we already have a stub for this function, recycle it.
+ return state.getFunctionToLazyStubMap().lookup(F);
+}
+
+/// getFunctionStub - This returns a pointer to a function stub, creating
+/// one on demand as needed.
+void *JITResolver::getLazyFunctionStub(Function *F) {
+ MutexGuard locked(TheJIT->lock);
+
+ // If we already have a lazy stub for this function, recycle it.
+ void *&Stub = state.getFunctionToLazyStubMap()[F];
+ if (Stub) return Stub;
+
+ // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we
+ // must resolve the symbol now.
+ void *Actual = TheJIT->isCompilingLazily()
+ ? (void *)(intptr_t)LazyResolverFn : (void *)nullptr;
+
+ // If this is an external declaration, attempt to resolve the address now
+ // to place in the stub.
+ if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) {
+ Actual = TheJIT->getPointerToFunction(F);
+
+ // If we resolved the symbol to a null address (eg. a weak external)
+ // don't emit a stub. Return a null pointer to the application.
+ if (!Actual) return nullptr;
+ }
+
+ TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout();
+ JE.startGVStub(F, SL.Size, SL.Alignment);
+ // Codegen a new stub, calling the lazy resolver or the actual address of the
+ // external function, if it was resolved.
+ Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE);
+ JE.finishGVStub();
+
+ if (Actual != (void*)(intptr_t)LazyResolverFn) {
+ // If we are getting the stub for an external function, we really want the
+ // address of the stub in the GlobalAddressMap for the JIT, not the address
+ // of the external function.
+ TheJIT->updateGlobalMapping(F, Stub);
+ }
+
+ DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '"
+ << F->getName() << "'\n");
+
+ if (TheJIT->isCompilingLazily()) {
+ // Register this JITResolver as the one corresponding to this call site so
+ // JITCompilerFn will be able to find it.
+ StubToResolverMap->RegisterStubResolver(Stub, this);
+
+ // Finally, keep track of the stub-to-Function mapping so that the
+ // JITCompilerFn knows which function to compile!
+ state.AddCallSite(Stub, F);
+ } else if (!Actual) {
+ // If we are JIT'ing non-lazily but need to call a function that does not
+ // exist yet, add it to the JIT's work list so that we can fill in the
+ // stub address later.
+ assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() &&
+ "'Actual' should have been set above.");
+ TheJIT->addPendingFunction(F);
+ }
+
+ return Stub;
+}
+
+/// getGlobalValueIndirectSym - Return a lazy pointer containing the specified
+/// GV address.
+void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) {
+ MutexGuard locked(TheJIT->lock);
+
+ // If we already have a stub for this global variable, recycle it.
+ void *&IndirectSym = state.getGlobalToIndirectSymMap()[GV];
+ if (IndirectSym) return IndirectSym;
+
+ // Otherwise, codegen a new indirect symbol.
+ IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress,
+ JE);
+
+ DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym
+ << "] for GV '" << GV->getName() << "'\n");
+
+ return IndirectSym;
+}
+
+/// getExternalFunctionStub - Return a stub for the function at the
+/// specified address, created lazily on demand.
+void *JITResolver::getExternalFunctionStub(void *FnAddr) {
+ // If we already have a stub for this function, recycle it.
+ void *&Stub = ExternalFnToStubMap[FnAddr];
+ if (Stub) return Stub;
+
+ TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout();
+ JE.startGVStub(nullptr, SL.Size, SL.Alignment);
+ Stub = TheJIT->getJITInfo().emitFunctionStub(nullptr, FnAddr, JE);
+ JE.finishGVStub();
+
+ DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub
+ << "] for external function at '" << FnAddr << "'\n");
+ return Stub;
+}
+
+unsigned JITResolver::getGOTIndexForAddr(void* addr) {
+ unsigned idx = revGOTMap[addr];
+ if (!idx) {
+ idx = ++nextGOTIndex;
+ revGOTMap[addr] = idx;
+ DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr ["
+ << addr << "]\n");
+ }
+ return idx;
+}
+
+/// JITCompilerFn - This function is called when a lazy compilation stub has
+/// been entered. It looks up which function this stub corresponds to, compiles
+/// it if necessary, then returns the resultant function pointer.
+void *JITResolver::JITCompilerFn(void *Stub) {
+ JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub);
+ assert(JR && "Unable to find the corresponding JITResolver to the call site");
+
+ Function* F = nullptr;
+ void* ActualPtr = nullptr;
+
+ {
+ // Only lock for getting the Function. The call getPointerToFunction made
+ // in this function might trigger function materializing, which requires
+ // JIT lock to be unlocked.
+ MutexGuard locked(JR->TheJIT->lock);
+
+ // The address given to us for the stub may not be exactly right, it might
+ // be a little bit after the stub. As such, use upper_bound to find it.
+ std::pair<void*, Function*> I =
+ JR->state.LookupFunctionFromCallSite(Stub);
+ F = I.second;
+ ActualPtr = I.first;
+ }
+
+ // If we have already code generated the function, just return the address.
+ void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F);
+
+ if (!Result) {
+ // Otherwise we don't have it, do lazy compilation now.
+
+ // If lazy compilation is disabled, emit a useful error message and abort.
+ if (!JR->TheJIT->isCompilingLazily()) {
+ report_fatal_error("LLVM JIT requested to do lazy compilation of"
+ " function '"
+ + F->getName() + "' when lazy compiles are disabled!");
+ }
+
+ DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName()
+ << "' In stub ptr = " << Stub << " actual ptr = "
+ << ActualPtr << "\n");
+ (void)ActualPtr;
+
+ Result = JR->TheJIT->getPointerToFunction(F);
+ }
+
+ // Reacquire the lock to update the GOT map.
+ MutexGuard locked(JR->TheJIT->lock);
+
+ // We might like to remove the call site from the CallSiteToFunction map, but
+ // we can't do that! Multiple threads could be stuck, waiting to acquire the
+ // lock above. As soon as the 1st function finishes compiling the function,
+ // the next one will be released, and needs to be able to find the function it
+ // needs to call.
+
+ // FIXME: We could rewrite all references to this stub if we knew them.
+
+ // What we will do is set the compiled function address to map to the
+ // same GOT entry as the stub so that later clients may update the GOT
+ // if they see it still using the stub address.
+ // Note: this is done so the Resolver doesn't have to manage GOT memory
+ // Do this without allocating map space if the target isn't using a GOT
+ if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end())
+ JR->revGOTMap[Result] = JR->revGOTMap[Stub];
+
+ return Result;
+}
+
+//===----------------------------------------------------------------------===//
+// JITEmitter code.
+//
+
+static GlobalObject *getSimpleAliasee(Constant *C) {
+ C = C->stripPointerCasts();
+ return dyn_cast<GlobalObject>(C);
+}
+
+void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference,
+ bool MayNeedFarStub) {
+ if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
+ return TheJIT->getOrEmitGlobalVariable(GV);
+
+ if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) {
+ // We can only handle simple cases.
+ if (GlobalValue *GV = getSimpleAliasee(GA->getAliasee()))
+ return TheJIT->getPointerToGlobal(GV);
+ return nullptr;
+ }
+
+ // If we have already compiled the function, return a pointer to its body.
+ Function *F = cast<Function>(V);
+
+ void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F);
+ if (FnStub) {
+ // Return the function stub if it's already created. We do this first so
+ // that we're returning the same address for the function as any previous
+ // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be
+ // close enough to call.
+ return FnStub;
+ }
+
+ // If we know the target can handle arbitrary-distance calls, try to
+ // return a direct pointer.
+ if (!MayNeedFarStub) {
+ // If we have code, go ahead and return that.
+ void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F);
+ if (ResultPtr) return ResultPtr;
+
+ // If this is an external function pointer, we can force the JIT to
+ // 'compile' it, which really just adds it to the map.
+ if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage())
+ return TheJIT->getPointerToFunction(F);
+ }
+
+ // Otherwise, we may need a to emit a stub, and, conservatively, we always do
+ // so. Note that it's possible to return null from getLazyFunctionStub in the
+ // case of a weak extern that fails to resolve.
+ return Resolver.getLazyFunctionStub(F);
+}
+
+void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) {
+ // Make sure GV is emitted first, and create a stub containing the fully
+ // resolved address.
+ void *GVAddress = getPointerToGlobal(V, Reference, false);
+ void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress);
+ return StubAddr;
+}
+
+void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) {
+ if (DL.isUnknown()) return;
+ if (!BeforePrintingInsn) return;
+
+ const LLVMContext &Context = EmissionDetails.MF->getFunction()->getContext();
+
+ if (DL.getScope(Context) != nullptr && PrevDL != DL) {
+ JITEvent_EmittedFunctionDetails::LineStart NextLine;
+ NextLine.Address = getCurrentPCValue();
+ NextLine.Loc = DL;
+ EmissionDetails.LineStarts.push_back(NextLine);
+ }
+
+ PrevDL = DL;
+}
+
+static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP,
+ const DataLayout *TD) {
+ const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants();
+ if (Constants.empty()) return 0;
+
+ unsigned Size = 0;
+ for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
+ MachineConstantPoolEntry CPE = Constants[i];
+ unsigned AlignMask = CPE.getAlignment() - 1;
+ Size = (Size + AlignMask) & ~AlignMask;
+ Type *Ty = CPE.getType();
+ Size += TD->getTypeAllocSize(Ty);
+ }
+ return Size;
+}
+
+void JITEmitter::startFunction(MachineFunction &F) {
+ DEBUG(dbgs() << "JIT: Starting CodeGen of Function "
+ << F.getName() << "\n");
+
+ uintptr_t ActualSize = 0;
+ // Set the memory writable, if it's not already
+ MemMgr->setMemoryWritable();
+
+ if (SizeEstimate > 0) {
+ // SizeEstimate will be non-zero on reallocation attempts.
+ ActualSize = SizeEstimate;
+ }
+
+ BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(),
+ ActualSize);
+ BufferEnd = BufferBegin+ActualSize;
+ EmittedFunctions[F.getFunction()].FunctionBody = BufferBegin;
+
+ // Ensure the constant pool/jump table info is at least 4-byte aligned.
+ emitAlignment(16);
+
+ emitConstantPool(F.getConstantPool());
+ if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo())
+ initJumpTableInfo(MJTI);
+
+ // About to start emitting the machine code for the function.
+ emitAlignment(std::max(F.getFunction()->getAlignment(), 8U));
+ TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr);
+ EmittedFunctions[F.getFunction()].Code = CurBufferPtr;
+
+ MBBLocations.clear();
+
+ EmissionDetails.MF = &F;
+ EmissionDetails.LineStarts.clear();
+}
+
+bool JITEmitter::finishFunction(MachineFunction &F) {
+ if (CurBufferPtr == BufferEnd) {
+ // We must call endFunctionBody before retrying, because
+ // deallocateMemForFunction requires it.
+ MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr);
+ retryWithMoreMemory(F);
+ return true;
+ }
+
+ if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo())
+ emitJumpTableInfo(MJTI);
+
+ // FnStart is the start of the text, not the start of the constant pool and
+ // other per-function data.
+ uint8_t *FnStart =
+ (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction());
+
+ // FnEnd is the end of the function's machine code.
+ uint8_t *FnEnd = CurBufferPtr;
+
+ if (!Relocations.empty()) {
+ CurFn = F.getFunction();
+ NumRelos += Relocations.size();
+
+ // Resolve the relocations to concrete pointers.
+ for (unsigned i = 0, e = Relocations.size(); i != e; ++i) {
+ MachineRelocation &MR = Relocations[i];
+ void *ResultPtr = nullptr;
+ if (!MR.letTargetResolve()) {
+ if (MR.isExternalSymbol()) {
+ ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(),
+ false);
+ DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to ["
+ << ResultPtr << "]\n");
+
+ // If the target REALLY wants a stub for this function, emit it now.
+ if (MR.mayNeedFarStub()) {
+ ResultPtr = Resolver.getExternalFunctionStub(ResultPtr);
+ }
+ } else if (MR.isGlobalValue()) {
+ ResultPtr = getPointerToGlobal(MR.getGlobalValue(),
+ BufferBegin+MR.getMachineCodeOffset(),
+ MR.mayNeedFarStub());
+ } else if (MR.isIndirectSymbol()) {
+ ResultPtr = getPointerToGVIndirectSym(
+ MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset());
+ } else if (MR.isBasicBlock()) {
+ ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock());
+ } else if (MR.isConstantPoolIndex()) {
+ ResultPtr =
+ (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex());
+ } else {
+ assert(MR.isJumpTableIndex());
+ ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex());
+ }
+
+ MR.setResultPointer(ResultPtr);
+ }
+
+ // if we are managing the GOT and the relocation wants an index,
+ // give it one
+ if (MR.isGOTRelative() && MemMgr->isManagingGOT()) {
+ unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr);
+ MR.setGOTIndex(idx);
+ if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) {
+ DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr
+ << " pointing at " << ((void**)MemMgr->getGOTBase())[idx]
+ << "\n");
+ ((void**)MemMgr->getGOTBase())[idx] = ResultPtr;
+ }
+ }
+ }
+
+ CurFn = nullptr;
+ TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0],
+ Relocations.size(), MemMgr->getGOTBase());
+ }
+
+ // Update the GOT entry for F to point to the new code.
+ if (MemMgr->isManagingGOT()) {
+ unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin);
+ if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) {
+ DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin
+ << " pointing at " << ((void**)MemMgr->getGOTBase())[idx]
+ << "\n");
+ ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin;
+ }
+ }
+
+ // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for
+ // global variables that were referenced in the relocations.
+ MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr);
+
+ if (CurBufferPtr == BufferEnd) {
+ retryWithMoreMemory(F);
+ return true;
+ } else {
+ // Now that we've succeeded in emitting the function, reset the
+ // SizeEstimate back down to zero.
+ SizeEstimate = 0;
+ }
+
+ BufferBegin = CurBufferPtr = nullptr;
+ NumBytes += FnEnd-FnStart;
+
+ // Invalidate the icache if necessary.
+ sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart);
+
+ TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart,
+ EmissionDetails);
+
+ // Reset the previous debug location.
+ PrevDL = DebugLoc();
+
+ DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart
+ << "] Function: " << F.getName()
+ << ": " << (FnEnd-FnStart) << " bytes of text, "
+ << Relocations.size() << " relocations\n");
+
+ Relocations.clear();
+ ConstPoolAddresses.clear();
+
+ // Mark code region readable and executable if it's not so already.
+ MemMgr->setMemoryExecutable();
+
+ DEBUG({
+ dbgs() << "JIT: Binary code:\n";
+ uint8_t* q = FnStart;
+ for (int i = 0; q < FnEnd; q += 4, ++i) {
+ if (i == 4)
+ i = 0;
+ if (i == 0)
+ dbgs() << "JIT: " << (long)(q - FnStart) << ": ";
+ bool Done = false;
+ for (int j = 3; j >= 0; --j) {
+ if (q + j >= FnEnd)
+ Done = true;
+ else
+ dbgs() << (unsigned short)q[j];
+ }
+ if (Done)
+ break;
+ dbgs() << ' ';
+ if (i == 3)
+ dbgs() << '\n';
+ }
+ dbgs()<< '\n';
+ });
+
+ if (MMI)
+ MMI->EndFunction();
+
+ return false;
+}
+
+void JITEmitter::retryWithMoreMemory(MachineFunction &F) {
+ DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n");
+ Relocations.clear(); // Clear the old relocations or we'll reapply them.
+ ConstPoolAddresses.clear();
+ ++NumRetries;
+ deallocateMemForFunction(F.getFunction());
+ // Try again with at least twice as much free space.
+ SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin));
+
+ for (MachineFunction::iterator MBB = F.begin(), E = F.end(); MBB != E; ++MBB){
+ if (MBB->hasAddressTaken())
+ TheJIT->clearPointerToBasicBlock(MBB->getBasicBlock());
+ }
+}
+
+/// deallocateMemForFunction - Deallocate all memory for the specified
+/// function body. Also drop any references the function has to stubs.
+/// May be called while the Function is being destroyed inside ~Value().
+void JITEmitter::deallocateMemForFunction(const Function *F) {
+ ValueMap<const Function *, EmittedCode, EmittedFunctionConfig>::iterator
+ Emitted = EmittedFunctions.find(F);
+ if (Emitted != EmittedFunctions.end()) {
+ MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody);
+ TheJIT->NotifyFreeingMachineCode(Emitted->second.Code);
+
+ EmittedFunctions.erase(Emitted);
+ }
+}
+
+
+void *JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) {
+ if (BufferBegin)
+ return JITCodeEmitter::allocateSpace(Size, Alignment);
+
+ // create a new memory block if there is no active one.
+ // care must be taken so that BufferBegin is invalidated when a
+ // block is trimmed
+ BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment);
+ BufferEnd = BufferBegin+Size;
+ return CurBufferPtr;
+}
+
+void *JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) {
+ // Delegate this call through the memory manager.
+ return MemMgr->allocateGlobal(Size, Alignment);
+}
+
+void JITEmitter::emitConstantPool(MachineConstantPool *MCP) {
+ if (TheJIT->getJITInfo().hasCustomConstantPool())
+ return;
+
+ const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants();
+ if (Constants.empty()) return;
+
+ unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getDataLayout());
+ unsigned Align = MCP->getConstantPoolAlignment();
+ ConstantPoolBase = allocateSpace(Size, Align);
+ ConstantPool = MCP;
+
+ if (!ConstantPoolBase) return; // Buffer overflow.
+
+ DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase
+ << "] (size: " << Size << ", alignment: " << Align << ")\n");
+
+ // Initialize the memory for all of the constant pool entries.
+ unsigned Offset = 0;
+ for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
+ MachineConstantPoolEntry CPE = Constants[i];
+ unsigned AlignMask = CPE.getAlignment() - 1;
+ Offset = (Offset + AlignMask) & ~AlignMask;
+
+ uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset;
+ ConstPoolAddresses.push_back(CAddr);
+ if (CPE.isMachineConstantPoolEntry()) {
+ // FIXME: add support to lower machine constant pool values into bytes!
+ report_fatal_error("Initialize memory with machine specific constant pool"
+ "entry has not been implemented!");
+ }
+ TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr);
+ DEBUG(dbgs() << "JIT: CP" << i << " at [0x";
+ dbgs().write_hex(CAddr) << "]\n");
+
+ Type *Ty = CPE.Val.ConstVal->getType();
+ Offset += TheJIT->getDataLayout()->getTypeAllocSize(Ty);
+ }
+}
+
+void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) {
+ if (TheJIT->getJITInfo().hasCustomJumpTables())
+ return;
+ if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline)
+ return;
+
+ const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
+ if (JT.empty()) return;
+
+ unsigned NumEntries = 0;
+ for (unsigned i = 0, e = JT.size(); i != e; ++i)
+ NumEntries += JT[i].MBBs.size();
+
+ unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getDataLayout());
+
+ // Just allocate space for all the jump tables now. We will fix up the actual
+ // MBB entries in the tables after we emit the code for each block, since then
+ // we will know the final locations of the MBBs in memory.
+ JumpTable = MJTI;
+ JumpTableBase = allocateSpace(NumEntries * EntrySize,
+ MJTI->getEntryAlignment(*TheJIT->getDataLayout()));
+}
+
+void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) {
+ if (TheJIT->getJITInfo().hasCustomJumpTables())
+ return;
+
+ const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
+ if (JT.empty() || !JumpTableBase) return;
+
+
+ switch (MJTI->getEntryKind()) {
+ case MachineJumpTableInfo::EK_Inline:
+ return;
+ case MachineJumpTableInfo::EK_BlockAddress: {
+ // EK_BlockAddress - Each entry is a plain address of block, e.g.:
+ // .word LBB123
+ assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == sizeof(void*) &&
+ "Cross JIT'ing?");
+
+ // For each jump table, map each target in the jump table to the address of
+ // an emitted MachineBasicBlock.
+ intptr_t *SlotPtr = (intptr_t*)JumpTableBase;
+
+ for (unsigned i = 0, e = JT.size(); i != e; ++i) {
+ const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs;
+ // Store the address of the basic block for this jump table slot in the
+ // memory we allocated for the jump table in 'initJumpTableInfo'
+ for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi)
+ *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]);
+ }
+ break;
+ }
+
+ case MachineJumpTableInfo::EK_Custom32:
+ case MachineJumpTableInfo::EK_GPRel32BlockAddress:
+ case MachineJumpTableInfo::EK_LabelDifference32: {
+ assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == 4&&"Cross JIT'ing?");
+ // For each jump table, place the offset from the beginning of the table
+ // to the target address.
+ int *SlotPtr = (int*)JumpTableBase;
+
+ for (unsigned i = 0, e = JT.size(); i != e; ++i) {
+ const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs;
+ // Store the offset of the basic block for this jump table slot in the
+ // memory we allocated for the jump table in 'initJumpTableInfo'
+ uintptr_t Base = (uintptr_t)SlotPtr;
+ for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) {
+ uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]);
+ /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook.
+ *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base);
+ }
+ }
+ break;
+ }
+ case MachineJumpTableInfo::EK_GPRel64BlockAddress:
+ llvm_unreachable(
+ "JT Info emission not implemented for GPRel64BlockAddress yet.");
+ }
+}
+
+void JITEmitter::startGVStub(const GlobalValue* GV,
+ unsigned StubSize, unsigned Alignment) {
+ SavedBufferBegin = BufferBegin;
+ SavedBufferEnd = BufferEnd;
+ SavedCurBufferPtr = CurBufferPtr;
+
+ BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment);
+ BufferEnd = BufferBegin+StubSize+1;
+}
+
+void JITEmitter::startGVStub(void *Buffer, unsigned StubSize) {
+ SavedBufferBegin = BufferBegin;
+ SavedBufferEnd = BufferEnd;
+ SavedCurBufferPtr = CurBufferPtr;
+
+ BufferBegin = CurBufferPtr = (uint8_t *)Buffer;
+ BufferEnd = BufferBegin+StubSize+1;
+}
+
+void JITEmitter::finishGVStub() {
+ assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space.");
+ NumBytes += getCurrentPCOffset();
+ BufferBegin = SavedBufferBegin;
+ BufferEnd = SavedBufferEnd;
+ CurBufferPtr = SavedCurBufferPtr;
+}
+
+void *JITEmitter::allocIndirectGV(const GlobalValue *GV,
+ const uint8_t *Buffer, size_t Size,
+ unsigned Alignment) {
+ uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment);
+ memcpy(IndGV, Buffer, Size);
+ return IndGV;
+}
+
+// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry
+// in the constant pool that was last emitted with the 'emitConstantPool'
+// method.
+//
+uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const {
+ assert(ConstantNum < ConstantPool->getConstants().size() &&
+ "Invalid ConstantPoolIndex!");
+ return ConstPoolAddresses[ConstantNum];
+}
+
+// getJumpTableEntryAddress - Return the address of the JumpTable with index
+// 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo'
+//
+uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const {
+ const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables();
+ assert(Index < JT.size() && "Invalid jump table index!");
+
+ unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getDataLayout());
+
+ unsigned Offset = 0;
+ for (unsigned i = 0; i < Index; ++i)
+ Offset += JT[i].MBBs.size();
+
+ Offset *= EntrySize;
+
+ return (uintptr_t)((char *)JumpTableBase + Offset);
+}
+
+void JITEmitter::EmittedFunctionConfig::onDelete(
+ JITEmitter *Emitter, const Function *F) {
+ Emitter->deallocateMemForFunction(F);
+}
+void JITEmitter::EmittedFunctionConfig::onRAUW(
+ JITEmitter *, const Function*, const Function*) {
+ llvm_unreachable("The JIT doesn't know how to handle a"
+ " RAUW on a value it has emitted.");
+}
+
+
+//===----------------------------------------------------------------------===//
+// Public interface to this file
+//===----------------------------------------------------------------------===//
+
+JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM,
+ TargetMachine &tm) {
+ return new JITEmitter(jit, JMM, tm);
+}
+
+// getPointerToFunctionOrStub - If the specified function has been
+// code-gen'd, return a pointer to the function. If not, compile it, or use
+// a stub to implement lazy compilation if available.
+//
+void *JIT::getPointerToFunctionOrStub(Function *F) {
+ // If we have already code generated the function, just return the address.
+ if (void *Addr = getPointerToGlobalIfAvailable(F))
+ return Addr;
+
+ // Get a stub if the target supports it.
+ JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter());
+ return JE->getJITResolver().getLazyFunctionStub(F);
+}
+
+void JIT::updateFunctionStubUnlocked(Function *F) {
+ // Get the empty stub we generated earlier.
+ JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter());
+ void *Stub = JE->getJITResolver().getLazyFunctionStub(F);
+ void *Addr = getPointerToGlobalIfAvailable(F);
+ assert(Addr != Stub && "Function must have non-stub address to be updated.");
+
+ // Tell the target jit info to rewrite the stub at the specified address,
+ // rather than creating a new one.
+ TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout();
+ JE->startGVStub(Stub, layout.Size);
+ getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter());
+ JE->finishGVStub();
+}
+
+/// freeMachineCodeForFunction - release machine code memory for given Function.
+///
+void JIT::freeMachineCodeForFunction(Function *F) {
+ // Delete translation for this from the ExecutionEngine, so it will get
+ // retranslated next time it is used.
+ updateGlobalMapping(F, nullptr);
+
+ // Free the actual memory for the function body and related stuff.
+ static_cast<JITEmitter*>(JCE)->deallocateMemForFunction(F);
+}
--- /dev/null
+//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the DefaultJITMemoryManager class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/JITMemoryManager.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Config/config.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Memory.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+#include <climits>
+#include <cstring>
+#include <vector>
+
+#if defined(__linux__)
+#if defined(HAVE_SYS_STAT_H)
+#include <sys/stat.h>
+#endif
+#include <fcntl.h>
+#include <unistd.h>
+#endif
+
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+STATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT");
+
+JITMemoryManager::~JITMemoryManager() {}
+
+//===----------------------------------------------------------------------===//
+// Memory Block Implementation.
+//===----------------------------------------------------------------------===//
+
+namespace {
+ /// MemoryRangeHeader - For a range of memory, this is the header that we put
+ /// on the block of memory. It is carefully crafted to be one word of memory.
+ /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader
+ /// which starts with this.
+ struct FreeRangeHeader;
+ struct MemoryRangeHeader {
+ /// ThisAllocated - This is true if this block is currently allocated. If
+ /// not, this can be converted to a FreeRangeHeader.
+ unsigned ThisAllocated : 1;
+
+ /// PrevAllocated - Keep track of whether the block immediately before us is
+ /// allocated. If not, the word immediately before this header is the size
+ /// of the previous block.
+ unsigned PrevAllocated : 1;
+
+ /// BlockSize - This is the size in bytes of this memory block,
+ /// including this header.
+ uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2);
+
+
+ /// getBlockAfter - Return the memory block immediately after this one.
+ ///
+ MemoryRangeHeader &getBlockAfter() const {
+ return *reinterpret_cast<MemoryRangeHeader *>(
+ reinterpret_cast<char*>(
+ const_cast<MemoryRangeHeader *>(this))+BlockSize);
+ }
+
+ /// getFreeBlockBefore - If the block before this one is free, return it,
+ /// otherwise return null.
+ FreeRangeHeader *getFreeBlockBefore() const {
+ if (PrevAllocated) return nullptr;
+ intptr_t PrevSize = reinterpret_cast<intptr_t *>(
+ const_cast<MemoryRangeHeader *>(this))[-1];
+ return reinterpret_cast<FreeRangeHeader *>(
+ reinterpret_cast<char*>(
+ const_cast<MemoryRangeHeader *>(this))-PrevSize);
+ }
+
+ /// FreeBlock - Turn an allocated block into a free block, adjusting
+ /// bits in the object headers, and adding an end of region memory block.
+ FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList);
+
+ /// TrimAllocationToSize - If this allocated block is significantly larger
+ /// than NewSize, split it into two pieces (where the former is NewSize
+ /// bytes, including the header), and add the new block to the free list.
+ FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList,
+ uint64_t NewSize);
+ };
+
+ /// FreeRangeHeader - For a memory block that isn't already allocated, this
+ /// keeps track of the current block and has a pointer to the next free block.
+ /// Free blocks are kept on a circularly linked list.
+ struct FreeRangeHeader : public MemoryRangeHeader {
+ FreeRangeHeader *Prev;
+ FreeRangeHeader *Next;
+
+ /// getMinBlockSize - Get the minimum size for a memory block. Blocks
+ /// smaller than this size cannot be created.
+ static unsigned getMinBlockSize() {
+ return sizeof(FreeRangeHeader)+sizeof(intptr_t);
+ }
+
+ /// SetEndOfBlockSizeMarker - The word at the end of every free block is
+ /// known to be the size of the free block. Set it for this block.
+ void SetEndOfBlockSizeMarker() {
+ void *EndOfBlock = (char*)this + BlockSize;
+ ((intptr_t *)EndOfBlock)[-1] = BlockSize;
+ }
+
+ FreeRangeHeader *RemoveFromFreeList() {
+ assert(Next->Prev == this && Prev->Next == this && "Freelist broken!");
+ Next->Prev = Prev;
+ return Prev->Next = Next;
+ }
+
+ void AddToFreeList(FreeRangeHeader *FreeList) {
+ Next = FreeList;
+ Prev = FreeList->Prev;
+ Prev->Next = this;
+ Next->Prev = this;
+ }
+
+ /// GrowBlock - The block after this block just got deallocated. Merge it
+ /// into the current block.
+ void GrowBlock(uintptr_t NewSize);
+
+ /// AllocateBlock - Mark this entire block allocated, updating freelists
+ /// etc. This returns a pointer to the circular free-list.
+ FreeRangeHeader *AllocateBlock();
+ };
+}
+
+
+/// AllocateBlock - Mark this entire block allocated, updating freelists
+/// etc. This returns a pointer to the circular free-list.
+FreeRangeHeader *FreeRangeHeader::AllocateBlock() {
+ assert(!ThisAllocated && !getBlockAfter().PrevAllocated &&
+ "Cannot allocate an allocated block!");
+ // Mark this block allocated.
+ ThisAllocated = 1;
+ getBlockAfter().PrevAllocated = 1;
+
+ // Remove it from the free list.
+ return RemoveFromFreeList();
+}
+
+/// FreeBlock - Turn an allocated block into a free block, adjusting
+/// bits in the object headers, and adding an end of region memory block.
+/// If possible, coalesce this block with neighboring blocks. Return the
+/// FreeRangeHeader to allocate from.
+FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) {
+ MemoryRangeHeader *FollowingBlock = &getBlockAfter();
+ assert(ThisAllocated && "This block is already free!");
+ assert(FollowingBlock->PrevAllocated && "Flags out of sync!");
+
+ FreeRangeHeader *FreeListToReturn = FreeList;
+
+ // If the block after this one is free, merge it into this block.
+ if (!FollowingBlock->ThisAllocated) {
+ FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock;
+ // "FreeList" always needs to be a valid free block. If we're about to
+ // coalesce with it, update our notion of what the free list is.
+ if (&FollowingFreeBlock == FreeList) {
+ FreeList = FollowingFreeBlock.Next;
+ FreeListToReturn = nullptr;
+ assert(&FollowingFreeBlock != FreeList && "No tombstone block?");
+ }
+ FollowingFreeBlock.RemoveFromFreeList();
+
+ // Include the following block into this one.
+ BlockSize += FollowingFreeBlock.BlockSize;
+ FollowingBlock = &FollowingFreeBlock.getBlockAfter();
+
+ // Tell the block after the block we are coalescing that this block is
+ // allocated.
+ FollowingBlock->PrevAllocated = 1;
+ }
+
+ assert(FollowingBlock->ThisAllocated && "Missed coalescing?");
+
+ if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) {
+ PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize);
+ return FreeListToReturn ? FreeListToReturn : PrevFreeBlock;
+ }
+
+ // Otherwise, mark this block free.
+ FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this;
+ FollowingBlock->PrevAllocated = 0;
+ FreeBlock.ThisAllocated = 0;
+
+ // Link this into the linked list of free blocks.
+ FreeBlock.AddToFreeList(FreeList);
+
+ // Add a marker at the end of the block, indicating the size of this free
+ // block.
+ FreeBlock.SetEndOfBlockSizeMarker();
+ return FreeListToReturn ? FreeListToReturn : &FreeBlock;
+}
+
+/// GrowBlock - The block after this block just got deallocated. Merge it
+/// into the current block.
+void FreeRangeHeader::GrowBlock(uintptr_t NewSize) {
+ assert(NewSize > BlockSize && "Not growing block?");
+ BlockSize = NewSize;
+ SetEndOfBlockSizeMarker();
+ getBlockAfter().PrevAllocated = 0;
+}
+
+/// TrimAllocationToSize - If this allocated block is significantly larger
+/// than NewSize, split it into two pieces (where the former is NewSize
+/// bytes, including the header), and add the new block to the free list.
+FreeRangeHeader *MemoryRangeHeader::
+TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) {
+ assert(ThisAllocated && getBlockAfter().PrevAllocated &&
+ "Cannot deallocate part of an allocated block!");
+
+ // Don't allow blocks to be trimmed below minimum required size
+ NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize);
+
+ // Round up size for alignment of header.
+ unsigned HeaderAlign = __alignof(FreeRangeHeader);
+ NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1);
+
+ // Size is now the size of the block we will remove from the start of the
+ // current block.
+ assert(NewSize <= BlockSize &&
+ "Allocating more space from this block than exists!");
+
+ // If splitting this block will cause the remainder to be too small, do not
+ // split the block.
+ if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize())
+ return FreeList;
+
+ // Otherwise, we splice the required number of bytes out of this block, form
+ // a new block immediately after it, then mark this block allocated.
+ MemoryRangeHeader &FormerNextBlock = getBlockAfter();
+
+ // Change the size of this block.
+ BlockSize = NewSize;
+
+ // Get the new block we just sliced out and turn it into a free block.
+ FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter();
+ NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock;
+ NewNextBlock.ThisAllocated = 0;
+ NewNextBlock.PrevAllocated = 1;
+ NewNextBlock.SetEndOfBlockSizeMarker();
+ FormerNextBlock.PrevAllocated = 0;
+ NewNextBlock.AddToFreeList(FreeList);
+ return &NewNextBlock;
+}
+
+//===----------------------------------------------------------------------===//
+// Memory Block Implementation.
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+ class DefaultJITMemoryManager;
+
+ class JITAllocator {
+ DefaultJITMemoryManager &JMM;
+ public:
+ JITAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { }
+ void *Allocate(size_t Size, size_t /*Alignment*/);
+ void Deallocate(void *Slab, size_t Size);
+ };
+
+ /// DefaultJITMemoryManager - Manage memory for the JIT code generation.
+ /// This splits a large block of MAP_NORESERVE'd memory into two
+ /// sections, one for function stubs, one for the functions themselves. We
+ /// have to do this because we may need to emit a function stub while in the
+ /// middle of emitting a function, and we don't know how large the function we
+ /// are emitting is.
+ class DefaultJITMemoryManager : public JITMemoryManager {
+ public:
+ /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at
+ /// least this much unless more is requested. Currently, in 512k slabs.
+ static const size_t DefaultCodeSlabSize = 512 * 1024;
+
+ /// DefaultSlabSize - Allocate globals and stubs into slabs of 64K (probably
+ /// 16 pages) unless we get an allocation above SizeThreshold.
+ static const size_t DefaultSlabSize = 64 * 1024;
+
+ /// DefaultSizeThreshold - For any allocation larger than 16K (probably
+ /// 4 pages), we should allocate a separate slab to avoid wasted space at
+ /// the end of a normal slab.
+ static const size_t DefaultSizeThreshold = 16 * 1024;
+
+ private:
+ // Whether to poison freed memory.
+ bool PoisonMemory;
+
+ /// LastSlab - This points to the last slab allocated and is used as the
+ /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all
+ /// stubs, data, and code contiguously in memory. In general, however, this
+ /// is not possible because the NearBlock parameter is ignored on Windows
+ /// platforms and even on Unix it works on a best-effort pasis.
+ sys::MemoryBlock LastSlab;
+
+ // Memory slabs allocated by the JIT. We refer to them as slabs so we don't
+ // confuse them with the blocks of memory described above.
+ std::vector<sys::MemoryBlock> CodeSlabs;
+ BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize,
+ DefaultSizeThreshold> StubAllocator;
+ BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize,
+ DefaultSizeThreshold> DataAllocator;
+
+ // Circular list of free blocks.
+ FreeRangeHeader *FreeMemoryList;
+
+ // When emitting code into a memory block, this is the block.
+ MemoryRangeHeader *CurBlock;
+
+ uint8_t *GOTBase; // Target Specific reserved memory
+ public:
+ DefaultJITMemoryManager();
+ ~DefaultJITMemoryManager();
+
+ /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the
+ /// last slab it allocated, so that subsequent allocations follow it.
+ sys::MemoryBlock allocateNewSlab(size_t size);
+
+ /// getPointerToNamedFunction - This method returns the address of the
+ /// specified function by using the dlsym function call.
+ void *getPointerToNamedFunction(const std::string &Name,
+ bool AbortOnFailure = true) override;
+
+ void AllocateGOT() override;
+
+ // Testing methods.
+ bool CheckInvariants(std::string &ErrorStr) override;
+ size_t GetDefaultCodeSlabSize() override { return DefaultCodeSlabSize; }
+ size_t GetDefaultDataSlabSize() override { return DefaultSlabSize; }
+ size_t GetDefaultStubSlabSize() override { return DefaultSlabSize; }
+ unsigned GetNumCodeSlabs() override { return CodeSlabs.size(); }
+ unsigned GetNumDataSlabs() override { return DataAllocator.GetNumSlabs(); }
+ unsigned GetNumStubSlabs() override { return StubAllocator.GetNumSlabs(); }
+
+ /// startFunctionBody - When a function starts, allocate a block of free
+ /// executable memory, returning a pointer to it and its actual size.
+ uint8_t *startFunctionBody(const Function *F,
+ uintptr_t &ActualSize) override {
+
+ FreeRangeHeader* candidateBlock = FreeMemoryList;
+ FreeRangeHeader* head = FreeMemoryList;
+ FreeRangeHeader* iter = head->Next;
+
+ uintptr_t largest = candidateBlock->BlockSize;
+
+ // Search for the largest free block
+ while (iter != head) {
+ if (iter->BlockSize > largest) {
+ largest = iter->BlockSize;
+ candidateBlock = iter;
+ }
+ iter = iter->Next;
+ }
+
+ largest = largest - sizeof(MemoryRangeHeader);
+
+ // If this block isn't big enough for the allocation desired, allocate
+ // another block of memory and add it to the free list.
+ if (largest < ActualSize ||
+ largest <= FreeRangeHeader::getMinBlockSize()) {
+ DEBUG(dbgs() << "JIT: Allocating another slab of memory for function.");
+ candidateBlock = allocateNewCodeSlab((size_t)ActualSize);
+ }
+
+ // Select this candidate block for allocation
+ CurBlock = candidateBlock;
+
+ // Allocate the entire memory block.
+ FreeMemoryList = candidateBlock->AllocateBlock();
+ ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader);
+ return (uint8_t *)(CurBlock + 1);
+ }
+
+ /// allocateNewCodeSlab - Helper method to allocate a new slab of code
+ /// memory from the OS and add it to the free list. Returns the new
+ /// FreeRangeHeader at the base of the slab.
+ FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) {
+ // If the user needs at least MinSize free memory, then we account for
+ // two MemoryRangeHeaders: the one in the user's block, and the one at the
+ // end of the slab.
+ size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader);
+ size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin);
+ sys::MemoryBlock B = allocateNewSlab(SlabSize);
+ CodeSlabs.push_back(B);
+ char *MemBase = (char*)(B.base());
+
+ // Put a tiny allocated block at the end of the memory chunk, so when
+ // FreeBlock calls getBlockAfter it doesn't fall off the end.
+ MemoryRangeHeader *EndBlock =
+ (MemoryRangeHeader*)(MemBase + B.size()) - 1;
+ EndBlock->ThisAllocated = 1;
+ EndBlock->PrevAllocated = 0;
+ EndBlock->BlockSize = sizeof(MemoryRangeHeader);
+
+ // Start out with a vast new block of free memory.
+ FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase;
+ NewBlock->ThisAllocated = 0;
+ // Make sure getFreeBlockBefore doesn't look into unmapped memory.
+ NewBlock->PrevAllocated = 1;
+ NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock;
+ NewBlock->SetEndOfBlockSizeMarker();
+ NewBlock->AddToFreeList(FreeMemoryList);
+
+ assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize &&
+ "The block was too small!");
+ return NewBlock;
+ }
+
+ /// endFunctionBody - The function F is now allocated, and takes the memory
+ /// in the range [FunctionStart,FunctionEnd).
+ void endFunctionBody(const Function *F, uint8_t *FunctionStart,
+ uint8_t *FunctionEnd) override {
+ assert(FunctionEnd > FunctionStart);
+ assert(FunctionStart == (uint8_t *)(CurBlock+1) &&
+ "Mismatched function start/end!");
+
+ uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock;
+
+ // Release the memory at the end of this block that isn't needed.
+ FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
+ }
+
+ /// allocateSpace - Allocate a memory block of the given size. This method
+ /// cannot be called between calls to startFunctionBody and endFunctionBody.
+ uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) override {
+ CurBlock = FreeMemoryList;
+ FreeMemoryList = FreeMemoryList->AllocateBlock();
+
+ uint8_t *result = (uint8_t *)(CurBlock + 1);
+
+ if (Alignment == 0) Alignment = 1;
+ result = (uint8_t*)(((intptr_t)result+Alignment-1) &
+ ~(intptr_t)(Alignment-1));
+
+ uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock;
+ FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
+
+ return result;
+ }
+
+ /// allocateStub - Allocate memory for a function stub.
+ uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize,
+ unsigned Alignment) override {
+ return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment);
+ }
+
+ /// allocateGlobal - Allocate memory for a global.
+ uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) override {
+ return (uint8_t*)DataAllocator.Allocate(Size, Alignment);
+ }
+
+ /// allocateCodeSection - Allocate memory for a code section.
+ uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
+ unsigned SectionID,
+ StringRef SectionName) override {
+ // Grow the required block size to account for the block header
+ Size += sizeof(*CurBlock);
+
+ // Alignment handling.
+ if (!Alignment)
+ Alignment = 16;
+ Size += Alignment - 1;
+
+ FreeRangeHeader* candidateBlock = FreeMemoryList;
+ FreeRangeHeader* head = FreeMemoryList;
+ FreeRangeHeader* iter = head->Next;
+
+ uintptr_t largest = candidateBlock->BlockSize;
+
+ // Search for the largest free block.
+ while (iter != head) {
+ if (iter->BlockSize > largest) {
+ largest = iter->BlockSize;
+ candidateBlock = iter;
+ }
+ iter = iter->Next;
+ }
+
+ largest = largest - sizeof(MemoryRangeHeader);
+
+ // If this block isn't big enough for the allocation desired, allocate
+ // another block of memory and add it to the free list.
+ if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) {
+ DEBUG(dbgs() << "JIT: Allocating another slab of memory for function.");
+ candidateBlock = allocateNewCodeSlab((size_t)Size);
+ }
+
+ // Select this candidate block for allocation
+ CurBlock = candidateBlock;
+
+ // Allocate the entire memory block.
+ FreeMemoryList = candidateBlock->AllocateBlock();
+ // Release the memory at the end of this block that isn't needed.
+ FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size);
+ uintptr_t unalignedAddr = (uintptr_t)CurBlock + sizeof(*CurBlock);
+ return (uint8_t*)RoundUpToAlignment((uint64_t)unalignedAddr, Alignment);
+ }
+
+ /// allocateDataSection - Allocate memory for a data section.
+ uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
+ unsigned SectionID, StringRef SectionName,
+ bool IsReadOnly) override {
+ return (uint8_t*)DataAllocator.Allocate(Size, Alignment);
+ }
+
+ bool finalizeMemory(std::string *ErrMsg) override {
+ return false;
+ }
+
+ uint8_t *getGOTBase() const override {
+ return GOTBase;
+ }
+
+ void deallocateBlock(void *Block) {
+ // Find the block that is allocated for this function.
+ MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1;
+ assert(MemRange->ThisAllocated && "Block isn't allocated!");
+
+ // Fill the buffer with garbage!
+ if (PoisonMemory) {
+ memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
+ }
+
+ // Free the memory.
+ FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
+ }
+
+ /// deallocateFunctionBody - Deallocate all memory for the specified
+ /// function body.
+ void deallocateFunctionBody(void *Body) override {
+ if (Body) deallocateBlock(Body);
+ }
+
+ /// setMemoryWritable - When code generation is in progress,
+ /// the code pages may need permissions changed.
+ void setMemoryWritable() override {
+ for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
+ sys::Memory::setWritable(CodeSlabs[i]);
+ }
+ /// setMemoryExecutable - When code generation is done and we're ready to
+ /// start execution, the code pages may need permissions changed.
+ void setMemoryExecutable() override {
+ for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
+ sys::Memory::setExecutable(CodeSlabs[i]);
+ }
+
+ /// setPoisonMemory - Controls whether we write garbage over freed memory.
+ ///
+ void setPoisonMemory(bool poison) override {
+ PoisonMemory = poison;
+ }
+ };
+}
+
+void *JITAllocator::Allocate(size_t Size, size_t /*Alignment*/) {
+ sys::MemoryBlock B = JMM.allocateNewSlab(Size);
+ return B.base();
+}
+
+void JITAllocator::Deallocate(void *Slab, size_t Size) {
+ sys::MemoryBlock B(Slab, Size);
+ sys::Memory::ReleaseRWX(B);
+}
+
+DefaultJITMemoryManager::DefaultJITMemoryManager()
+ :
+#ifdef NDEBUG
+ PoisonMemory(false),
+#else
+ PoisonMemory(true),
+#endif
+ LastSlab(nullptr, 0), StubAllocator(*this), DataAllocator(*this) {
+
+ // Allocate space for code.
+ sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize);
+ CodeSlabs.push_back(MemBlock);
+ uint8_t *MemBase = (uint8_t*)MemBlock.base();
+
+ // We set up the memory chunk with 4 mem regions, like this:
+ // [ START
+ // [ Free #0 ] -> Large space to allocate functions from.
+ // [ Allocated #1 ] -> Tiny space to separate regions.
+ // [ Free #2 ] -> Tiny space so there is always at least 1 free block.
+ // [ Allocated #3 ] -> Tiny space to prevent looking past end of block.
+ // END ]
+ //
+ // The last three blocks are never deallocated or touched.
+
+ // Add MemoryRangeHeader to the end of the memory region, indicating that
+ // the space after the block of memory is allocated. This is block #3.
+ MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1;
+ Mem3->ThisAllocated = 1;
+ Mem3->PrevAllocated = 0;
+ Mem3->BlockSize = sizeof(MemoryRangeHeader);
+
+ /// Add a tiny free region so that the free list always has one entry.
+ FreeRangeHeader *Mem2 =
+ (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize());
+ Mem2->ThisAllocated = 0;
+ Mem2->PrevAllocated = 1;
+ Mem2->BlockSize = FreeRangeHeader::getMinBlockSize();
+ Mem2->SetEndOfBlockSizeMarker();
+ Mem2->Prev = Mem2; // Mem2 *is* the free list for now.
+ Mem2->Next = Mem2;
+
+ /// Add a tiny allocated region so that Mem2 is never coalesced away.
+ MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1;
+ Mem1->ThisAllocated = 1;
+ Mem1->PrevAllocated = 0;
+ Mem1->BlockSize = sizeof(MemoryRangeHeader);
+
+ // Add a FreeRangeHeader to the start of the function body region, indicating
+ // that the space is free. Mark the previous block allocated so we never look
+ // at it.
+ FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase;
+ Mem0->ThisAllocated = 0;
+ Mem0->PrevAllocated = 1;
+ Mem0->BlockSize = (char*)Mem1-(char*)Mem0;
+ Mem0->SetEndOfBlockSizeMarker();
+ Mem0->AddToFreeList(Mem2);
+
+ // Start out with the freelist pointing to Mem0.
+ FreeMemoryList = Mem0;
+
+ GOTBase = nullptr;
+}
+
+void DefaultJITMemoryManager::AllocateGOT() {
+ assert(!GOTBase && "Cannot allocate the got multiple times");
+ GOTBase = new uint8_t[sizeof(void*) * 8192];
+ HasGOT = true;
+}
+
+DefaultJITMemoryManager::~DefaultJITMemoryManager() {
+ for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
+ sys::Memory::ReleaseRWX(CodeSlabs[i]);
+
+ delete[] GOTBase;
+}
+
+sys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) {
+ // Allocate a new block close to the last one.
+ std::string ErrMsg;
+ sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : nullptr;
+ sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg);
+ if (!B.base()) {
+ report_fatal_error("Allocation failed when allocating new memory in the"
+ " JIT\n" + Twine(ErrMsg));
+ }
+ LastSlab = B;
+ ++NumSlabs;
+ // Initialize the slab to garbage when debugging.
+ if (PoisonMemory) {
+ memset(B.base(), 0xCD, B.size());
+ }
+ return B;
+}
+
+/// CheckInvariants - For testing only. Return "" if all internal invariants
+/// are preserved, and a helpful error message otherwise. For free and
+/// allocated blocks, make sure that adding BlockSize gives a valid block.
+/// For free blocks, make sure they're in the free list and that their end of
+/// block size marker is correct. This function should return an error before
+/// accessing bad memory. This function is defined here instead of in
+/// JITMemoryManagerTest.cpp so that we don't have to expose all of the
+/// implementation details of DefaultJITMemoryManager.
+bool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) {
+ raw_string_ostream Err(ErrorStr);
+
+ // Construct a the set of FreeRangeHeader pointers so we can query it
+ // efficiently.
+ llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet;
+ FreeRangeHeader* FreeHead = FreeMemoryList;
+ FreeRangeHeader* FreeRange = FreeHead;
+
+ do {
+ // Check that the free range pointer is in the blocks we've allocated.
+ bool Found = false;
+ for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(),
+ E = CodeSlabs.end(); I != E && !Found; ++I) {
+ char *Start = (char*)I->base();
+ char *End = Start + I->size();
+ Found = (Start <= (char*)FreeRange && (char*)FreeRange < End);
+ }
+ if (!Found) {
+ Err << "Corrupt free list; points to " << FreeRange;
+ return false;
+ }
+
+ if (FreeRange->Next->Prev != FreeRange) {
+ Err << "Next and Prev pointers do not match.";
+ return false;
+ }
+
+ // Otherwise, add it to the set.
+ FreeHdrSet.insert(FreeRange);
+ FreeRange = FreeRange->Next;
+ } while (FreeRange != FreeHead);
+
+ // Go over each block, and look at each MemoryRangeHeader.
+ for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(),
+ E = CodeSlabs.end(); I != E; ++I) {
+ char *Start = (char*)I->base();
+ char *End = Start + I->size();
+
+ // Check each memory range.
+ for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = nullptr;
+ Start <= (char*)Hdr && (char*)Hdr < End;
+ Hdr = &Hdr->getBlockAfter()) {
+ if (Hdr->ThisAllocated == 0) {
+ // Check that this range is in the free list.
+ if (!FreeHdrSet.count(Hdr)) {
+ Err << "Found free header at " << Hdr << " that is not in free list.";
+ return false;
+ }
+
+ // Now make sure the size marker at the end of the block is correct.
+ uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1;
+ if (!(Start <= (char*)Marker && (char*)Marker < End)) {
+ Err << "Block size in header points out of current MemoryBlock.";
+ return false;
+ }
+ if (Hdr->BlockSize != *Marker) {
+ Err << "End of block size marker (" << *Marker << ") "
+ << "and BlockSize (" << Hdr->BlockSize << ") don't match.";
+ return false;
+ }
+ }
+
+ if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) {
+ Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != "
+ << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")";
+ return false;
+ } else if (!LastHdr && !Hdr->PrevAllocated) {
+ Err << "The first header should have PrevAllocated true.";
+ return false;
+ }
+
+ // Remember the last header.
+ LastHdr = Hdr;
+ }
+ }
+
+ // All invariants are preserved.
+ return true;
+}
+
+//===----------------------------------------------------------------------===//
+// getPointerToNamedFunction() implementation.
+//===----------------------------------------------------------------------===//
+
+// AtExitHandlers - List of functions to call when the program exits,
+// registered with the atexit() library function.
+static std::vector<void (*)()> AtExitHandlers;
+
+/// runAtExitHandlers - Run any functions registered by the program's
+/// calls to atexit(3), which we intercept and store in
+/// AtExitHandlers.
+///
+static void runAtExitHandlers() {
+ while (!AtExitHandlers.empty()) {
+ void (*Fn)() = AtExitHandlers.back();
+ AtExitHandlers.pop_back();
+ Fn();
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// Function stubs that are invoked instead of certain library calls
+//
+// Force the following functions to be linked in to anything that uses the
+// JIT. This is a hack designed to work around the all-too-clever Glibc
+// strategy of making these functions work differently when inlined vs. when
+// not inlined, and hiding their real definitions in a separate archive file
+// that the dynamic linker can't see. For more info, search for
+// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274.
+#if defined(__linux__) && defined(__GLIBC__)
+/* stat functions are redirecting to __xstat with a version number. On x86-64
+ * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat'
+ * available as an exported symbol, so we have to add it explicitly.
+ */
+namespace {
+class StatSymbols {
+public:
+ StatSymbols() {
+ sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat);
+ sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat);
+ sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat);
+ sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64);
+ sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64);
+ sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64);
+ sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64);
+ sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64);
+ sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64);
+ sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit);
+ sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod);
+ }
+};
+}
+static StatSymbols initStatSymbols;
+#endif // __linux__
+
+// jit_exit - Used to intercept the "exit" library call.
+static void jit_exit(int Status) {
+ runAtExitHandlers(); // Run atexit handlers...
+ exit(Status);
+}
+
+// jit_atexit - Used to intercept the "atexit" library call.
+static int jit_atexit(void (*Fn)()) {
+ AtExitHandlers.push_back(Fn); // Take note of atexit handler...
+ return 0; // Always successful
+}
+
+static int jit_noop() {
+ return 0;
+}
+
+//===----------------------------------------------------------------------===//
+//
+/// getPointerToNamedFunction - This method returns the address of the specified
+/// function by using the dynamic loader interface. As such it is only useful
+/// for resolving library symbols, not code generated symbols.
+///
+void *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name,
+ bool AbortOnFailure) {
+ // Check to see if this is one of the functions we want to intercept. Note,
+ // we cast to intptr_t here to silence a -pedantic warning that complains
+ // about casting a function pointer to a normal pointer.
+ if (Name == "exit") return (void*)(intptr_t)&jit_exit;
+ if (Name == "atexit") return (void*)(intptr_t)&jit_atexit;
+
+ // We should not invoke parent's ctors/dtors from generated main()!
+ // On Mingw and Cygwin, the symbol __main is resolved to
+ // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors
+ // (and register wrong callee's dtors with atexit(3)).
+ // We expect ExecutionEngine::runStaticConstructorsDestructors()
+ // is called before ExecutionEngine::runFunctionAsMain() is called.
+ if (Name == "__main") return (void*)(intptr_t)&jit_noop;
+
+ const char *NameStr = Name.c_str();
+ // If this is an asm specifier, skip the sentinal.
+ if (NameStr[0] == 1) ++NameStr;
+
+ // If it's an external function, look it up in the process image...
+ void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr);
+ if (Ptr) return Ptr;
+
+ // If it wasn't found and if it starts with an underscore ('_') character,
+ // try again without the underscore.
+ if (NameStr[0] == '_') {
+ Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1);
+ if (Ptr) return Ptr;
+ }
+
+ // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These
+ // are references to hidden visibility symbols that dlsym cannot resolve.
+ // If we have one of these, strip off $LDBLStub and try again.
+#if defined(__APPLE__) && defined(__ppc__)
+ if (Name.size() > 9 && Name[Name.size()-9] == '$' &&
+ memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) {
+ // First try turning $LDBLStub into $LDBL128. If that fails, strip it off.
+ // This mirrors logic in libSystemStubs.a.
+ std::string Prefix = std::string(Name.begin(), Name.end()-9);
+ if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false))
+ return Ptr;
+ if (void *Ptr = getPointerToNamedFunction(Prefix, false))
+ return Ptr;
+ }
+#endif
+
+ if (AbortOnFailure) {
+ report_fatal_error("Program used external function '"+Name+
+ "' which could not be resolved!");
+ }
+ return nullptr;
+}
+
+
+
+JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() {
+ return new DefaultJITMemoryManager();
+}
+
+const size_t DefaultJITMemoryManager::DefaultCodeSlabSize;
+const size_t DefaultJITMemoryManager::DefaultSlabSize;
+const size_t DefaultJITMemoryManager::DefaultSizeThreshold;
--- /dev/null
+;===- ./lib/ExecutionEngine/JIT/LLVMBuild.txt ------------------*- Conf -*--===;
+;
+; The LLVM Compiler Infrastructure
+;
+; This file is distributed under the University of Illinois Open Source
+; License. See LICENSE.TXT for details.
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+; http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = JIT
+parent = ExecutionEngine
+required_libraries = CodeGen Core ExecutionEngine Support
--- /dev/null
+##===- lib/ExecutionEngine/JIT/Makefile --------------------*- Makefile -*-===##
+#
+# The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../../..
+LIBRARYNAME = LLVMJIT
+
+# Get the $(ARCH) setting
+include $(LEVEL)/Makefile.config
+
+# Enable the X86 JIT if compiling on X86
+ifeq ($(ARCH), x86)
+ ENABLE_X86_JIT = 1
+endif
+
+# This flag can also be used on the command line to force inclusion
+# of the X86 JIT on non-X86 hosts
+ifdef ENABLE_X86_JIT
+ CPPFLAGS += -DENABLE_X86_JIT
+endif
+
+# Enable the Sparc JIT if compiling on Sparc
+ifeq ($(ARCH), Sparc)
+ ENABLE_SPARC_JIT = 1
+endif
+
+# This flag can also be used on the command line to force inclusion
+# of the Sparc JIT on non-Sparc hosts
+ifdef ENABLE_SPARC_JIT
+ CPPFLAGS += -DENABLE_SPARC_JIT
+endif
+
+include $(LEVEL)/Makefile.common
;===------------------------------------------------------------------------===;
[common]
-subdirectories = Interpreter MCJIT RuntimeDyld IntelJITEvents OProfileJIT
+subdirectories = Interpreter JIT MCJIT RuntimeDyld IntelJITEvents OProfileJIT
[component_0]
type = Library
add_llvm_library(LLVMMCJIT
- JITMemoryManager.cpp
MCJIT.cpp
SectionMemoryManager.cpp
)
+++ /dev/null
-//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the DefaultJITMemoryManager class.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ExecutionEngine/JITMemoryManager.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Config/config.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/Support/Allocator.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/DynamicLibrary.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/Memory.h"
-#include "llvm/Support/raw_ostream.h"
-#include <cassert>
-#include <climits>
-#include <cstring>
-#include <vector>
-
-#if defined(__linux__)
-#if defined(HAVE_SYS_STAT_H)
-#include <sys/stat.h>
-#endif
-#include <fcntl.h>
-#include <unistd.h>
-#endif
-
-using namespace llvm;
-
-#define DEBUG_TYPE "jit"
-
-STATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT");
-
-JITMemoryManager::~JITMemoryManager() {}
-
-//===----------------------------------------------------------------------===//
-// Memory Block Implementation.
-//===----------------------------------------------------------------------===//
-
-namespace {
- /// MemoryRangeHeader - For a range of memory, this is the header that we put
- /// on the block of memory. It is carefully crafted to be one word of memory.
- /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader
- /// which starts with this.
- struct FreeRangeHeader;
- struct MemoryRangeHeader {
- /// ThisAllocated - This is true if this block is currently allocated. If
- /// not, this can be converted to a FreeRangeHeader.
- unsigned ThisAllocated : 1;
-
- /// PrevAllocated - Keep track of whether the block immediately before us is
- /// allocated. If not, the word immediately before this header is the size
- /// of the previous block.
- unsigned PrevAllocated : 1;
-
- /// BlockSize - This is the size in bytes of this memory block,
- /// including this header.
- uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2);
-
-
- /// getBlockAfter - Return the memory block immediately after this one.
- ///
- MemoryRangeHeader &getBlockAfter() const {
- return *reinterpret_cast<MemoryRangeHeader *>(
- reinterpret_cast<char*>(
- const_cast<MemoryRangeHeader *>(this))+BlockSize);
- }
-
- /// getFreeBlockBefore - If the block before this one is free, return it,
- /// otherwise return null.
- FreeRangeHeader *getFreeBlockBefore() const {
- if (PrevAllocated) return nullptr;
- intptr_t PrevSize = reinterpret_cast<intptr_t *>(
- const_cast<MemoryRangeHeader *>(this))[-1];
- return reinterpret_cast<FreeRangeHeader *>(
- reinterpret_cast<char*>(
- const_cast<MemoryRangeHeader *>(this))-PrevSize);
- }
-
- /// FreeBlock - Turn an allocated block into a free block, adjusting
- /// bits in the object headers, and adding an end of region memory block.
- FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList);
-
- /// TrimAllocationToSize - If this allocated block is significantly larger
- /// than NewSize, split it into two pieces (where the former is NewSize
- /// bytes, including the header), and add the new block to the free list.
- FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList,
- uint64_t NewSize);
- };
-
- /// FreeRangeHeader - For a memory block that isn't already allocated, this
- /// keeps track of the current block and has a pointer to the next free block.
- /// Free blocks are kept on a circularly linked list.
- struct FreeRangeHeader : public MemoryRangeHeader {
- FreeRangeHeader *Prev;
- FreeRangeHeader *Next;
-
- /// getMinBlockSize - Get the minimum size for a memory block. Blocks
- /// smaller than this size cannot be created.
- static unsigned getMinBlockSize() {
- return sizeof(FreeRangeHeader)+sizeof(intptr_t);
- }
-
- /// SetEndOfBlockSizeMarker - The word at the end of every free block is
- /// known to be the size of the free block. Set it for this block.
- void SetEndOfBlockSizeMarker() {
- void *EndOfBlock = (char*)this + BlockSize;
- ((intptr_t *)EndOfBlock)[-1] = BlockSize;
- }
-
- FreeRangeHeader *RemoveFromFreeList() {
- assert(Next->Prev == this && Prev->Next == this && "Freelist broken!");
- Next->Prev = Prev;
- return Prev->Next = Next;
- }
-
- void AddToFreeList(FreeRangeHeader *FreeList) {
- Next = FreeList;
- Prev = FreeList->Prev;
- Prev->Next = this;
- Next->Prev = this;
- }
-
- /// GrowBlock - The block after this block just got deallocated. Merge it
- /// into the current block.
- void GrowBlock(uintptr_t NewSize);
-
- /// AllocateBlock - Mark this entire block allocated, updating freelists
- /// etc. This returns a pointer to the circular free-list.
- FreeRangeHeader *AllocateBlock();
- };
-}
-
-
-/// AllocateBlock - Mark this entire block allocated, updating freelists
-/// etc. This returns a pointer to the circular free-list.
-FreeRangeHeader *FreeRangeHeader::AllocateBlock() {
- assert(!ThisAllocated && !getBlockAfter().PrevAllocated &&
- "Cannot allocate an allocated block!");
- // Mark this block allocated.
- ThisAllocated = 1;
- getBlockAfter().PrevAllocated = 1;
-
- // Remove it from the free list.
- return RemoveFromFreeList();
-}
-
-/// FreeBlock - Turn an allocated block into a free block, adjusting
-/// bits in the object headers, and adding an end of region memory block.
-/// If possible, coalesce this block with neighboring blocks. Return the
-/// FreeRangeHeader to allocate from.
-FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) {
- MemoryRangeHeader *FollowingBlock = &getBlockAfter();
- assert(ThisAllocated && "This block is already free!");
- assert(FollowingBlock->PrevAllocated && "Flags out of sync!");
-
- FreeRangeHeader *FreeListToReturn = FreeList;
-
- // If the block after this one is free, merge it into this block.
- if (!FollowingBlock->ThisAllocated) {
- FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock;
- // "FreeList" always needs to be a valid free block. If we're about to
- // coalesce with it, update our notion of what the free list is.
- if (&FollowingFreeBlock == FreeList) {
- FreeList = FollowingFreeBlock.Next;
- FreeListToReturn = nullptr;
- assert(&FollowingFreeBlock != FreeList && "No tombstone block?");
- }
- FollowingFreeBlock.RemoveFromFreeList();
-
- // Include the following block into this one.
- BlockSize += FollowingFreeBlock.BlockSize;
- FollowingBlock = &FollowingFreeBlock.getBlockAfter();
-
- // Tell the block after the block we are coalescing that this block is
- // allocated.
- FollowingBlock->PrevAllocated = 1;
- }
-
- assert(FollowingBlock->ThisAllocated && "Missed coalescing?");
-
- if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) {
- PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize);
- return FreeListToReturn ? FreeListToReturn : PrevFreeBlock;
- }
-
- // Otherwise, mark this block free.
- FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this;
- FollowingBlock->PrevAllocated = 0;
- FreeBlock.ThisAllocated = 0;
-
- // Link this into the linked list of free blocks.
- FreeBlock.AddToFreeList(FreeList);
-
- // Add a marker at the end of the block, indicating the size of this free
- // block.
- FreeBlock.SetEndOfBlockSizeMarker();
- return FreeListToReturn ? FreeListToReturn : &FreeBlock;
-}
-
-/// GrowBlock - The block after this block just got deallocated. Merge it
-/// into the current block.
-void FreeRangeHeader::GrowBlock(uintptr_t NewSize) {
- assert(NewSize > BlockSize && "Not growing block?");
- BlockSize = NewSize;
- SetEndOfBlockSizeMarker();
- getBlockAfter().PrevAllocated = 0;
-}
-
-/// TrimAllocationToSize - If this allocated block is significantly larger
-/// than NewSize, split it into two pieces (where the former is NewSize
-/// bytes, including the header), and add the new block to the free list.
-FreeRangeHeader *MemoryRangeHeader::
-TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) {
- assert(ThisAllocated && getBlockAfter().PrevAllocated &&
- "Cannot deallocate part of an allocated block!");
-
- // Don't allow blocks to be trimmed below minimum required size
- NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize);
-
- // Round up size for alignment of header.
- unsigned HeaderAlign = __alignof(FreeRangeHeader);
- NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1);
-
- // Size is now the size of the block we will remove from the start of the
- // current block.
- assert(NewSize <= BlockSize &&
- "Allocating more space from this block than exists!");
-
- // If splitting this block will cause the remainder to be too small, do not
- // split the block.
- if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize())
- return FreeList;
-
- // Otherwise, we splice the required number of bytes out of this block, form
- // a new block immediately after it, then mark this block allocated.
- MemoryRangeHeader &FormerNextBlock = getBlockAfter();
-
- // Change the size of this block.
- BlockSize = NewSize;
-
- // Get the new block we just sliced out and turn it into a free block.
- FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter();
- NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock;
- NewNextBlock.ThisAllocated = 0;
- NewNextBlock.PrevAllocated = 1;
- NewNextBlock.SetEndOfBlockSizeMarker();
- FormerNextBlock.PrevAllocated = 0;
- NewNextBlock.AddToFreeList(FreeList);
- return &NewNextBlock;
-}
-
-//===----------------------------------------------------------------------===//
-// Memory Block Implementation.
-//===----------------------------------------------------------------------===//
-
-namespace {
-
- class DefaultJITMemoryManager;
-
- class JITAllocator {
- DefaultJITMemoryManager &JMM;
- public:
- JITAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { }
- void *Allocate(size_t Size, size_t /*Alignment*/);
- void Deallocate(void *Slab, size_t Size);
- };
-
- /// DefaultJITMemoryManager - Manage memory for the JIT code generation.
- /// This splits a large block of MAP_NORESERVE'd memory into two
- /// sections, one for function stubs, one for the functions themselves. We
- /// have to do this because we may need to emit a function stub while in the
- /// middle of emitting a function, and we don't know how large the function we
- /// are emitting is.
- class DefaultJITMemoryManager : public JITMemoryManager {
- public:
- /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at
- /// least this much unless more is requested. Currently, in 512k slabs.
- static const size_t DefaultCodeSlabSize = 512 * 1024;
-
- /// DefaultSlabSize - Allocate globals and stubs into slabs of 64K (probably
- /// 16 pages) unless we get an allocation above SizeThreshold.
- static const size_t DefaultSlabSize = 64 * 1024;
-
- /// DefaultSizeThreshold - For any allocation larger than 16K (probably
- /// 4 pages), we should allocate a separate slab to avoid wasted space at
- /// the end of a normal slab.
- static const size_t DefaultSizeThreshold = 16 * 1024;
-
- private:
- // Whether to poison freed memory.
- bool PoisonMemory;
-
- /// LastSlab - This points to the last slab allocated and is used as the
- /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all
- /// stubs, data, and code contiguously in memory. In general, however, this
- /// is not possible because the NearBlock parameter is ignored on Windows
- /// platforms and even on Unix it works on a best-effort pasis.
- sys::MemoryBlock LastSlab;
-
- // Memory slabs allocated by the JIT. We refer to them as slabs so we don't
- // confuse them with the blocks of memory described above.
- std::vector<sys::MemoryBlock> CodeSlabs;
- BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize,
- DefaultSizeThreshold> StubAllocator;
- BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize,
- DefaultSizeThreshold> DataAllocator;
-
- // Circular list of free blocks.
- FreeRangeHeader *FreeMemoryList;
-
- // When emitting code into a memory block, this is the block.
- MemoryRangeHeader *CurBlock;
-
- uint8_t *GOTBase; // Target Specific reserved memory
- public:
- DefaultJITMemoryManager();
- ~DefaultJITMemoryManager();
-
- /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the
- /// last slab it allocated, so that subsequent allocations follow it.
- sys::MemoryBlock allocateNewSlab(size_t size);
-
- /// getPointerToNamedFunction - This method returns the address of the
- /// specified function by using the dlsym function call.
- void *getPointerToNamedFunction(const std::string &Name,
- bool AbortOnFailure = true) override;
-
- void AllocateGOT() override;
-
- // Testing methods.
- bool CheckInvariants(std::string &ErrorStr) override;
- size_t GetDefaultCodeSlabSize() override { return DefaultCodeSlabSize; }
- size_t GetDefaultDataSlabSize() override { return DefaultSlabSize; }
- size_t GetDefaultStubSlabSize() override { return DefaultSlabSize; }
- unsigned GetNumCodeSlabs() override { return CodeSlabs.size(); }
- unsigned GetNumDataSlabs() override { return DataAllocator.GetNumSlabs(); }
- unsigned GetNumStubSlabs() override { return StubAllocator.GetNumSlabs(); }
-
- /// startFunctionBody - When a function starts, allocate a block of free
- /// executable memory, returning a pointer to it and its actual size.
- uint8_t *startFunctionBody(const Function *F,
- uintptr_t &ActualSize) override {
-
- FreeRangeHeader* candidateBlock = FreeMemoryList;
- FreeRangeHeader* head = FreeMemoryList;
- FreeRangeHeader* iter = head->Next;
-
- uintptr_t largest = candidateBlock->BlockSize;
-
- // Search for the largest free block
- while (iter != head) {
- if (iter->BlockSize > largest) {
- largest = iter->BlockSize;
- candidateBlock = iter;
- }
- iter = iter->Next;
- }
-
- largest = largest - sizeof(MemoryRangeHeader);
-
- // If this block isn't big enough for the allocation desired, allocate
- // another block of memory and add it to the free list.
- if (largest < ActualSize ||
- largest <= FreeRangeHeader::getMinBlockSize()) {
- DEBUG(dbgs() << "JIT: Allocating another slab of memory for function.");
- candidateBlock = allocateNewCodeSlab((size_t)ActualSize);
- }
-
- // Select this candidate block for allocation
- CurBlock = candidateBlock;
-
- // Allocate the entire memory block.
- FreeMemoryList = candidateBlock->AllocateBlock();
- ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader);
- return (uint8_t *)(CurBlock + 1);
- }
-
- /// allocateNewCodeSlab - Helper method to allocate a new slab of code
- /// memory from the OS and add it to the free list. Returns the new
- /// FreeRangeHeader at the base of the slab.
- FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) {
- // If the user needs at least MinSize free memory, then we account for
- // two MemoryRangeHeaders: the one in the user's block, and the one at the
- // end of the slab.
- size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader);
- size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin);
- sys::MemoryBlock B = allocateNewSlab(SlabSize);
- CodeSlabs.push_back(B);
- char *MemBase = (char*)(B.base());
-
- // Put a tiny allocated block at the end of the memory chunk, so when
- // FreeBlock calls getBlockAfter it doesn't fall off the end.
- MemoryRangeHeader *EndBlock =
- (MemoryRangeHeader*)(MemBase + B.size()) - 1;
- EndBlock->ThisAllocated = 1;
- EndBlock->PrevAllocated = 0;
- EndBlock->BlockSize = sizeof(MemoryRangeHeader);
-
- // Start out with a vast new block of free memory.
- FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase;
- NewBlock->ThisAllocated = 0;
- // Make sure getFreeBlockBefore doesn't look into unmapped memory.
- NewBlock->PrevAllocated = 1;
- NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock;
- NewBlock->SetEndOfBlockSizeMarker();
- NewBlock->AddToFreeList(FreeMemoryList);
-
- assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize &&
- "The block was too small!");
- return NewBlock;
- }
-
- /// endFunctionBody - The function F is now allocated, and takes the memory
- /// in the range [FunctionStart,FunctionEnd).
- void endFunctionBody(const Function *F, uint8_t *FunctionStart,
- uint8_t *FunctionEnd) override {
- assert(FunctionEnd > FunctionStart);
- assert(FunctionStart == (uint8_t *)(CurBlock+1) &&
- "Mismatched function start/end!");
-
- uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock;
-
- // Release the memory at the end of this block that isn't needed.
- FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
- }
-
- /// allocateSpace - Allocate a memory block of the given size. This method
- /// cannot be called between calls to startFunctionBody and endFunctionBody.
- uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) override {
- CurBlock = FreeMemoryList;
- FreeMemoryList = FreeMemoryList->AllocateBlock();
-
- uint8_t *result = (uint8_t *)(CurBlock + 1);
-
- if (Alignment == 0) Alignment = 1;
- result = (uint8_t*)(((intptr_t)result+Alignment-1) &
- ~(intptr_t)(Alignment-1));
-
- uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock;
- FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
-
- return result;
- }
-
- /// allocateStub - Allocate memory for a function stub.
- uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize,
- unsigned Alignment) override {
- return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment);
- }
-
- /// allocateGlobal - Allocate memory for a global.
- uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) override {
- return (uint8_t*)DataAllocator.Allocate(Size, Alignment);
- }
-
- /// allocateCodeSection - Allocate memory for a code section.
- uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
- unsigned SectionID,
- StringRef SectionName) override {
- // Grow the required block size to account for the block header
- Size += sizeof(*CurBlock);
-
- // Alignment handling.
- if (!Alignment)
- Alignment = 16;
- Size += Alignment - 1;
-
- FreeRangeHeader* candidateBlock = FreeMemoryList;
- FreeRangeHeader* head = FreeMemoryList;
- FreeRangeHeader* iter = head->Next;
-
- uintptr_t largest = candidateBlock->BlockSize;
-
- // Search for the largest free block.
- while (iter != head) {
- if (iter->BlockSize > largest) {
- largest = iter->BlockSize;
- candidateBlock = iter;
- }
- iter = iter->Next;
- }
-
- largest = largest - sizeof(MemoryRangeHeader);
-
- // If this block isn't big enough for the allocation desired, allocate
- // another block of memory and add it to the free list.
- if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) {
- DEBUG(dbgs() << "JIT: Allocating another slab of memory for function.");
- candidateBlock = allocateNewCodeSlab((size_t)Size);
- }
-
- // Select this candidate block for allocation
- CurBlock = candidateBlock;
-
- // Allocate the entire memory block.
- FreeMemoryList = candidateBlock->AllocateBlock();
- // Release the memory at the end of this block that isn't needed.
- FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size);
- uintptr_t unalignedAddr = (uintptr_t)CurBlock + sizeof(*CurBlock);
- return (uint8_t*)RoundUpToAlignment((uint64_t)unalignedAddr, Alignment);
- }
-
- /// allocateDataSection - Allocate memory for a data section.
- uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
- unsigned SectionID, StringRef SectionName,
- bool IsReadOnly) override {
- return (uint8_t*)DataAllocator.Allocate(Size, Alignment);
- }
-
- bool finalizeMemory(std::string *ErrMsg) override {
- return false;
- }
-
- uint8_t *getGOTBase() const override {
- return GOTBase;
- }
-
- void deallocateBlock(void *Block) {
- // Find the block that is allocated for this function.
- MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1;
- assert(MemRange->ThisAllocated && "Block isn't allocated!");
-
- // Fill the buffer with garbage!
- if (PoisonMemory) {
- memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
- }
-
- // Free the memory.
- FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
- }
-
- /// deallocateFunctionBody - Deallocate all memory for the specified
- /// function body.
- void deallocateFunctionBody(void *Body) override {
- if (Body) deallocateBlock(Body);
- }
-
- /// setMemoryWritable - When code generation is in progress,
- /// the code pages may need permissions changed.
- void setMemoryWritable() override {
- for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
- sys::Memory::setWritable(CodeSlabs[i]);
- }
- /// setMemoryExecutable - When code generation is done and we're ready to
- /// start execution, the code pages may need permissions changed.
- void setMemoryExecutable() override {
- for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
- sys::Memory::setExecutable(CodeSlabs[i]);
- }
-
- /// setPoisonMemory - Controls whether we write garbage over freed memory.
- ///
- void setPoisonMemory(bool poison) override {
- PoisonMemory = poison;
- }
- };
-}
-
-void *JITAllocator::Allocate(size_t Size, size_t /*Alignment*/) {
- sys::MemoryBlock B = JMM.allocateNewSlab(Size);
- return B.base();
-}
-
-void JITAllocator::Deallocate(void *Slab, size_t Size) {
- sys::MemoryBlock B(Slab, Size);
- sys::Memory::ReleaseRWX(B);
-}
-
-DefaultJITMemoryManager::DefaultJITMemoryManager()
- :
-#ifdef NDEBUG
- PoisonMemory(false),
-#else
- PoisonMemory(true),
-#endif
- LastSlab(nullptr, 0), StubAllocator(*this), DataAllocator(*this) {
-
- // Allocate space for code.
- sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize);
- CodeSlabs.push_back(MemBlock);
- uint8_t *MemBase = (uint8_t*)MemBlock.base();
-
- // We set up the memory chunk with 4 mem regions, like this:
- // [ START
- // [ Free #0 ] -> Large space to allocate functions from.
- // [ Allocated #1 ] -> Tiny space to separate regions.
- // [ Free #2 ] -> Tiny space so there is always at least 1 free block.
- // [ Allocated #3 ] -> Tiny space to prevent looking past end of block.
- // END ]
- //
- // The last three blocks are never deallocated or touched.
-
- // Add MemoryRangeHeader to the end of the memory region, indicating that
- // the space after the block of memory is allocated. This is block #3.
- MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1;
- Mem3->ThisAllocated = 1;
- Mem3->PrevAllocated = 0;
- Mem3->BlockSize = sizeof(MemoryRangeHeader);
-
- /// Add a tiny free region so that the free list always has one entry.
- FreeRangeHeader *Mem2 =
- (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize());
- Mem2->ThisAllocated = 0;
- Mem2->PrevAllocated = 1;
- Mem2->BlockSize = FreeRangeHeader::getMinBlockSize();
- Mem2->SetEndOfBlockSizeMarker();
- Mem2->Prev = Mem2; // Mem2 *is* the free list for now.
- Mem2->Next = Mem2;
-
- /// Add a tiny allocated region so that Mem2 is never coalesced away.
- MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1;
- Mem1->ThisAllocated = 1;
- Mem1->PrevAllocated = 0;
- Mem1->BlockSize = sizeof(MemoryRangeHeader);
-
- // Add a FreeRangeHeader to the start of the function body region, indicating
- // that the space is free. Mark the previous block allocated so we never look
- // at it.
- FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase;
- Mem0->ThisAllocated = 0;
- Mem0->PrevAllocated = 1;
- Mem0->BlockSize = (char*)Mem1-(char*)Mem0;
- Mem0->SetEndOfBlockSizeMarker();
- Mem0->AddToFreeList(Mem2);
-
- // Start out with the freelist pointing to Mem0.
- FreeMemoryList = Mem0;
-
- GOTBase = nullptr;
-}
-
-void DefaultJITMemoryManager::AllocateGOT() {
- assert(!GOTBase && "Cannot allocate the got multiple times");
- GOTBase = new uint8_t[sizeof(void*) * 8192];
- HasGOT = true;
-}
-
-DefaultJITMemoryManager::~DefaultJITMemoryManager() {
- for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i)
- sys::Memory::ReleaseRWX(CodeSlabs[i]);
-
- delete[] GOTBase;
-}
-
-sys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) {
- // Allocate a new block close to the last one.
- std::string ErrMsg;
- sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : nullptr;
- sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg);
- if (!B.base()) {
- report_fatal_error("Allocation failed when allocating new memory in the"
- " JIT\n" + Twine(ErrMsg));
- }
- LastSlab = B;
- ++NumSlabs;
- // Initialize the slab to garbage when debugging.
- if (PoisonMemory) {
- memset(B.base(), 0xCD, B.size());
- }
- return B;
-}
-
-/// CheckInvariants - For testing only. Return "" if all internal invariants
-/// are preserved, and a helpful error message otherwise. For free and
-/// allocated blocks, make sure that adding BlockSize gives a valid block.
-/// For free blocks, make sure they're in the free list and that their end of
-/// block size marker is correct. This function should return an error before
-/// accessing bad memory. This function is defined here instead of in
-/// JITMemoryManagerTest.cpp so that we don't have to expose all of the
-/// implementation details of DefaultJITMemoryManager.
-bool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) {
- raw_string_ostream Err(ErrorStr);
-
- // Construct a the set of FreeRangeHeader pointers so we can query it
- // efficiently.
- llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet;
- FreeRangeHeader* FreeHead = FreeMemoryList;
- FreeRangeHeader* FreeRange = FreeHead;
-
- do {
- // Check that the free range pointer is in the blocks we've allocated.
- bool Found = false;
- for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(),
- E = CodeSlabs.end(); I != E && !Found; ++I) {
- char *Start = (char*)I->base();
- char *End = Start + I->size();
- Found = (Start <= (char*)FreeRange && (char*)FreeRange < End);
- }
- if (!Found) {
- Err << "Corrupt free list; points to " << FreeRange;
- return false;
- }
-
- if (FreeRange->Next->Prev != FreeRange) {
- Err << "Next and Prev pointers do not match.";
- return false;
- }
-
- // Otherwise, add it to the set.
- FreeHdrSet.insert(FreeRange);
- FreeRange = FreeRange->Next;
- } while (FreeRange != FreeHead);
-
- // Go over each block, and look at each MemoryRangeHeader.
- for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(),
- E = CodeSlabs.end(); I != E; ++I) {
- char *Start = (char*)I->base();
- char *End = Start + I->size();
-
- // Check each memory range.
- for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = nullptr;
- Start <= (char*)Hdr && (char*)Hdr < End;
- Hdr = &Hdr->getBlockAfter()) {
- if (Hdr->ThisAllocated == 0) {
- // Check that this range is in the free list.
- if (!FreeHdrSet.count(Hdr)) {
- Err << "Found free header at " << Hdr << " that is not in free list.";
- return false;
- }
-
- // Now make sure the size marker at the end of the block is correct.
- uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1;
- if (!(Start <= (char*)Marker && (char*)Marker < End)) {
- Err << "Block size in header points out of current MemoryBlock.";
- return false;
- }
- if (Hdr->BlockSize != *Marker) {
- Err << "End of block size marker (" << *Marker << ") "
- << "and BlockSize (" << Hdr->BlockSize << ") don't match.";
- return false;
- }
- }
-
- if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) {
- Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != "
- << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")";
- return false;
- } else if (!LastHdr && !Hdr->PrevAllocated) {
- Err << "The first header should have PrevAllocated true.";
- return false;
- }
-
- // Remember the last header.
- LastHdr = Hdr;
- }
- }
-
- // All invariants are preserved.
- return true;
-}
-
-//===----------------------------------------------------------------------===//
-// getPointerToNamedFunction() implementation.
-//===----------------------------------------------------------------------===//
-
-// AtExitHandlers - List of functions to call when the program exits,
-// registered with the atexit() library function.
-static std::vector<void (*)()> AtExitHandlers;
-
-/// runAtExitHandlers - Run any functions registered by the program's
-/// calls to atexit(3), which we intercept and store in
-/// AtExitHandlers.
-///
-static void runAtExitHandlers() {
- while (!AtExitHandlers.empty()) {
- void (*Fn)() = AtExitHandlers.back();
- AtExitHandlers.pop_back();
- Fn();
- }
-}
-
-//===----------------------------------------------------------------------===//
-// Function stubs that are invoked instead of certain library calls
-//
-// Force the following functions to be linked in to anything that uses the
-// JIT. This is a hack designed to work around the all-too-clever Glibc
-// strategy of making these functions work differently when inlined vs. when
-// not inlined, and hiding their real definitions in a separate archive file
-// that the dynamic linker can't see. For more info, search for
-// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274.
-#if defined(__linux__) && defined(__GLIBC__)
-/* stat functions are redirecting to __xstat with a version number. On x86-64
- * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat'
- * available as an exported symbol, so we have to add it explicitly.
- */
-namespace {
-class StatSymbols {
-public:
- StatSymbols() {
- sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat);
- sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat);
- sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat);
- sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64);
- sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64);
- sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64);
- sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64);
- sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64);
- sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64);
- sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit);
- sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod);
- }
-};
-}
-static StatSymbols initStatSymbols;
-#endif // __linux__
-
-// jit_exit - Used to intercept the "exit" library call.
-static void jit_exit(int Status) {
- runAtExitHandlers(); // Run atexit handlers...
- exit(Status);
-}
-
-// jit_atexit - Used to intercept the "atexit" library call.
-static int jit_atexit(void (*Fn)()) {
- AtExitHandlers.push_back(Fn); // Take note of atexit handler...
- return 0; // Always successful
-}
-
-static int jit_noop() {
- return 0;
-}
-
-//===----------------------------------------------------------------------===//
-//
-/// getPointerToNamedFunction - This method returns the address of the specified
-/// function by using the dynamic loader interface. As such it is only useful
-/// for resolving library symbols, not code generated symbols.
-///
-void *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name,
- bool AbortOnFailure) {
- // Check to see if this is one of the functions we want to intercept. Note,
- // we cast to intptr_t here to silence a -pedantic warning that complains
- // about casting a function pointer to a normal pointer.
- if (Name == "exit") return (void*)(intptr_t)&jit_exit;
- if (Name == "atexit") return (void*)(intptr_t)&jit_atexit;
-
- // We should not invoke parent's ctors/dtors from generated main()!
- // On Mingw and Cygwin, the symbol __main is resolved to
- // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors
- // (and register wrong callee's dtors with atexit(3)).
- // We expect ExecutionEngine::runStaticConstructorsDestructors()
- // is called before ExecutionEngine::runFunctionAsMain() is called.
- if (Name == "__main") return (void*)(intptr_t)&jit_noop;
-
- const char *NameStr = Name.c_str();
- // If this is an asm specifier, skip the sentinal.
- if (NameStr[0] == 1) ++NameStr;
-
- // If it's an external function, look it up in the process image...
- void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr);
- if (Ptr) return Ptr;
-
- // If it wasn't found and if it starts with an underscore ('_') character,
- // try again without the underscore.
- if (NameStr[0] == '_') {
- Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1);
- if (Ptr) return Ptr;
- }
-
- // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These
- // are references to hidden visibility symbols that dlsym cannot resolve.
- // If we have one of these, strip off $LDBLStub and try again.
-#if defined(__APPLE__) && defined(__ppc__)
- if (Name.size() > 9 && Name[Name.size()-9] == '$' &&
- memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) {
- // First try turning $LDBLStub into $LDBL128. If that fails, strip it off.
- // This mirrors logic in libSystemStubs.a.
- std::string Prefix = std::string(Name.begin(), Name.end()-9);
- if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false))
- return Ptr;
- if (void *Ptr = getPointerToNamedFunction(Prefix, false))
- return Ptr;
- }
-#endif
-
- if (AbortOnFailure) {
- report_fatal_error("Program used external function '"+Name+
- "' which could not be resolved!");
- }
- return nullptr;
-}
-
-
-
-JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() {
- return new DefaultJITMemoryManager();
-}
-
-const size_t DefaultJITMemoryManager::DefaultCodeSlabSize;
-const size_t DefaultJITMemoryManager::DefaultSlabSize;
-const size_t DefaultJITMemoryManager::DefaultSizeThreshold;
finalizeLoadedModules();
}
+void *MCJIT::getPointerToBasicBlock(BasicBlock *BB) {
+ report_fatal_error("not yet implemented");
+}
+
uint64_t MCJIT::getExistingSymbolAddress(const std::string &Name) {
Mangler Mang(TM->getSubtargetImpl()->getDataLayout());
SmallString<128> FullName;
return (void*)Dyld.getSymbolLoadAddress(Name);
}
+void *MCJIT::recompileAndRelinkFunction(Function *F) {
+ report_fatal_error("not yet implemented");
+}
+
+void MCJIT::freeMachineCodeForFunction(Function *F) {
+ report_fatal_error("not yet implemented");
+}
+
void MCJIT::runStaticConstructorsDestructorsInModulePtrSet(
bool isDtors, ModulePtrSet::iterator I, ModulePtrSet::iterator E) {
for (; I != E; ++I) {
if (!L)
return;
MutexGuard locked(lock);
- auto I = std::find(EventListeners.rbegin(), EventListeners.rend(), L);
+ SmallVector<JITEventListener*, 2>::reverse_iterator I=
+ std::find(EventListeners.rbegin(), EventListeners.rend(), L);
if (I != EventListeners.rend()) {
std::swap(*I, EventListeners.back());
EventListeners.pop_back();
void MCJIT::NotifyFreeingObject(const ObjectImage& Obj) {
MutexGuard locked(lock);
for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
- JITEventListener *L = EventListeners[I];
- L->NotifyFreeingObject(Obj);
+ EventListeners[I]->NotifyFreeingObject(Obj);
}
}
MCContext *Ctx;
LinkingMemoryManager MemMgr;
RuntimeDyld Dyld;
- std::vector<JITEventListener*> EventListeners;
+ SmallVector<JITEventListener*, 2> EventListeners;
OwningModuleContainer OwnedModules;
/// \param isDtors - Run the destructors instead of constructors.
void runStaticConstructorsDestructors(bool isDtors) override;
+ void *getPointerToBasicBlock(BasicBlock *BB) override;
+
void *getPointerToFunction(Function *F) override;
+ void *recompileAndRelinkFunction(Function *F) override;
+
+ void freeMachineCodeForFunction(Function *F) override;
+
GenericValue runFunction(Function *F,
const std::vector<GenericValue> &ArgValues) override;
include $(LEVEL)/Makefile.config
-PARALLEL_DIRS = Interpreter MCJIT RuntimeDyld
+PARALLEL_DIRS = Interpreter JIT MCJIT RuntimeDyld
ifeq ($(USE_INTEL_JITEVENTS), 1)
PARALLEL_DIRS += IntelJITEvents
// MCJIT can generate code for remote targets, but the old JIT and Interpreter
// must use the host architecture.
- if (WhichEngine != EngineKind::Interpreter && M)
+ if (UseMCJIT && WhichEngine != EngineKind::Interpreter && M)
TT.setTriple(M->getTargetTriple());
return selectTarget(TT, MArch, MCPU, MAttrs);
}
// FIXME: non-iOS ARM FastISel is broken with MCJIT.
- if (TheTriple.getArch() == Triple::arm &&
+ if (UseMCJIT &&
+ TheTriple.getArch() == Triple::arm &&
!TheTriple.isiOS() &&
OptLevel == CodeGenOpt::None) {
OptLevel = CodeGenOpt::Less;
tablegen(LLVM AArch64GenRegisterInfo.inc -gen-register-info)
tablegen(LLVM AArch64GenInstrInfo.inc -gen-instr-info)
-tablegen(LLVM AArch64GenMCCodeEmitter.inc -gen-emitter)
+tablegen(LLVM AArch64GenMCCodeEmitter.inc -gen-emitter -mc-emitter)
tablegen(LLVM AArch64GenMCPseudoLowering.inc -gen-pseudo-lowering)
tablegen(LLVM AArch64GenAsmWriter.inc -gen-asm-writer)
tablegen(LLVM AArch64GenAsmWriter1.inc -gen-asm-writer -asmwriternum=1)
class ARMBaseTargetMachine;
class FunctionPass;
class ImmutablePass;
+class JITCodeEmitter;
class MachineInstr;
class MCInst;
class TargetLowering;
FunctionPass *createARMISelDag(ARMBaseTargetMachine &TM,
CodeGenOpt::Level OptLevel);
+
+FunctionPass *createARMJITCodeEmitterPass(ARMBaseTargetMachine &TM,
+ JITCodeEmitter &JCE);
+
FunctionPass *createA15SDOptimizerPass();
FunctionPass *createARMLoadStoreOptimizationPass(bool PreAlloc = false);
FunctionPass *createARMExpandPseudoPass();
--- /dev/null
+//===-- ARM/ARMCodeEmitter.cpp - Convert ARM code to machine code ---------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the pass that transforms the ARM machine instructions into
+// relocatable machine code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ARM.h"
+#include "ARMBaseInstrInfo.h"
+#include "ARMConstantPoolValue.h"
+#include "ARMMachineFunctionInfo.h"
+#include "ARMRelocations.h"
+#include "ARMSubtarget.h"
+#include "ARMTargetMachine.h"
+#include "MCTargetDesc/ARMAddressingModes.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/JITCodeEmitter.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Function.h"
+#include "llvm/PassManager.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#ifndef NDEBUG
+#include <iomanip>
+#endif
+using namespace llvm;
+
+#define DEBUG_TYPE "jit"
+
+STATISTIC(NumEmitted, "Number of machine instructions emitted");
+
+namespace {
+
+ class ARMCodeEmitter : public MachineFunctionPass {
+ ARMJITInfo *JTI;
+ const ARMBaseInstrInfo *II;
+ const DataLayout *TD;
+ const ARMSubtarget *Subtarget;
+ TargetMachine &TM;
+ JITCodeEmitter &MCE;
+ MachineModuleInfo *MMI;
+ const std::vector<MachineConstantPoolEntry> *MCPEs;
+ const std::vector<MachineJumpTableEntry> *MJTEs;
+ bool IsPIC;
+ bool IsThumb;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<MachineModuleInfo>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ static char ID;
+ public:
+ ARMCodeEmitter(TargetMachine &tm, JITCodeEmitter &mce)
+ : MachineFunctionPass(ID), JTI(nullptr),
+ II((const ARMBaseInstrInfo *)tm.getSubtargetImpl()->getInstrInfo()),
+ TD(tm.getSubtargetImpl()->getDataLayout()), TM(tm), MCE(mce),
+ MCPEs(nullptr), MJTEs(nullptr),
+ IsPIC(TM.getRelocationModel() == Reloc::PIC_), IsThumb(false) {}
+
+ /// getBinaryCodeForInstr - This function, generated by the
+ /// CodeEmitterGenerator using TableGen, produces the binary encoding for
+ /// machine instructions.
+ uint64_t getBinaryCodeForInstr(const MachineInstr &MI) const;
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ const char *getPassName() const override {
+ return "ARM Machine Code Emitter";
+ }
+
+ void emitInstruction(const MachineInstr &MI);
+
+ private:
+
+ void emitWordLE(unsigned Binary);
+ void emitDWordLE(uint64_t Binary);
+ void emitConstPoolInstruction(const MachineInstr &MI);
+ void emitMOVi32immInstruction(const MachineInstr &MI);
+ void emitMOVi2piecesInstruction(const MachineInstr &MI);
+ void emitLEApcrelJTInstruction(const MachineInstr &MI);
+ void emitPseudoMoveInstruction(const MachineInstr &MI);
+ void addPCLabel(unsigned LabelID);
+ void emitPseudoInstruction(const MachineInstr &MI);
+ unsigned getMachineSoRegOpValue(const MachineInstr &MI,
+ const MCInstrDesc &MCID,
+ const MachineOperand &MO,
+ unsigned OpIdx);
+
+ unsigned getMachineSoImmOpValue(unsigned SoImm);
+ unsigned getAddrModeSBit(const MachineInstr &MI,
+ const MCInstrDesc &MCID) const;
+
+ void emitDataProcessingInstruction(const MachineInstr &MI,
+ unsigned ImplicitRd = 0,
+ unsigned ImplicitRn = 0);
+
+ void emitLoadStoreInstruction(const MachineInstr &MI,
+ unsigned ImplicitRd = 0,
+ unsigned ImplicitRn = 0);
+
+ void emitMiscLoadStoreInstruction(const MachineInstr &MI,
+ unsigned ImplicitRn = 0);
+
+ void emitLoadStoreMultipleInstruction(const MachineInstr &MI);
+
+ void emitMulFrmInstruction(const MachineInstr &MI);
+
+ void emitExtendInstruction(const MachineInstr &MI);
+
+ void emitMiscArithInstruction(const MachineInstr &MI);
+
+ void emitSaturateInstruction(const MachineInstr &MI);
+
+ void emitBranchInstruction(const MachineInstr &MI);
+
+ void emitInlineJumpTable(unsigned JTIndex);
+
+ void emitMiscBranchInstruction(const MachineInstr &MI);
+
+ void emitVFPArithInstruction(const MachineInstr &MI);
+
+ void emitVFPConversionInstruction(const MachineInstr &MI);
+
+ void emitVFPLoadStoreInstruction(const MachineInstr &MI);
+
+ void emitVFPLoadStoreMultipleInstruction(const MachineInstr &MI);
+
+ void emitNEONLaneInstruction(const MachineInstr &MI);
+ void emitNEONDupInstruction(const MachineInstr &MI);
+ void emitNEON1RegModImmInstruction(const MachineInstr &MI);
+ void emitNEON2RegInstruction(const MachineInstr &MI);
+ void emitNEON3RegInstruction(const MachineInstr &MI);
+
+ /// getMachineOpValue - Return binary encoding of operand. If the machine
+ /// operand requires relocation, record the relocation and return zero.
+ unsigned getMachineOpValue(const MachineInstr &MI,
+ const MachineOperand &MO) const;
+ unsigned getMachineOpValue(const MachineInstr &MI, unsigned OpIdx) const {
+ return getMachineOpValue(MI, MI.getOperand(OpIdx));
+ }
+
+ // FIXME: The legacy JIT ARMCodeEmitter doesn't rely on the the
+ // TableGen'erated getBinaryCodeForInstr() function to encode any
+ // operand values, instead querying getMachineOpValue() directly for
+ // each operand it needs to encode. Thus, any of the new encoder
+ // helper functions can simply return 0 as the values the return
+ // are already handled elsewhere. They are placeholders to allow this
+ // encoder to continue to function until the MC encoder is sufficiently
+ // far along that this one can be eliminated entirely.
+ unsigned NEONThumb2DataIPostEncoder(const MachineInstr &MI, unsigned Val)
+ const { return 0; }
+ unsigned NEONThumb2LoadStorePostEncoder(const MachineInstr &MI,unsigned Val)
+ const { return 0; }
+ unsigned NEONThumb2DupPostEncoder(const MachineInstr &MI,unsigned Val)
+ const { return 0; }
+ unsigned NEONThumb2V8PostEncoder(const MachineInstr &MI,unsigned Val)
+ const { return 0; }
+ unsigned VFPThumb2PostEncoder(const MachineInstr&MI, unsigned Val)
+ const { return 0; }
+ unsigned getAdrLabelOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbAdrLabelOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbBLTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbBLXTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbBRTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbBCCTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbCBTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getBranchTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getUnconditionalBranchTargetOpValue(const MachineInstr &MI,
+ unsigned Op) const { return 0; }
+ unsigned getARMBranchTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getARMBLTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getARMBLXTargetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getCCOutOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getSOImmOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2SOImmOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getSORegRegOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getSORegImmOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getThumbAddrModeRegRegOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2AddrModeImm8OpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2Imm8s4OpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2AddrModeImm8s4OpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2AddrModeImm0_1020s4OpValue(const MachineInstr &MI,unsigned Op)
+ const { return 0; }
+ unsigned getT2AddrModeImm8OffsetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2AddrModeSORegOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2SORegOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getT2AdrLabelOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getAddrMode6AddressOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getAddrMode6OneLane32AddressOpValue(const MachineInstr &MI,
+ unsigned Op)
+ const { return 0; }
+ unsigned getAddrMode6DupAddressOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getAddrMode6OffsetOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getBitfieldInvertedMaskOpValue(const MachineInstr &MI,
+ unsigned Op) const { return 0; }
+ uint32_t getLdStSORegOpValue(const MachineInstr &MI, unsigned OpIdx)
+ const { return 0; }
+
+ unsigned getAddrModeImm12OpValue(const MachineInstr &MI, unsigned Op)
+ const {
+ // {17-13} = reg
+ // {12} = (U)nsigned (add == '1', sub == '0')
+ // {11-0} = imm12
+ const MachineOperand &MO = MI.getOperand(Op);
+ const MachineOperand &MO1 = MI.getOperand(Op + 1);
+ if (!MO.isReg()) {
+ emitConstPoolAddress(MO.getIndex(), ARM::reloc_arm_cp_entry);
+ return 0;
+ }
+ unsigned Reg = II->getRegisterInfo().getEncodingValue(MO.getReg());
+ int32_t Imm12 = MO1.getImm();
+ uint32_t Binary;
+ Binary = Imm12 & 0xfff;
+ if (Imm12 >= 0)
+ Binary |= (1 << 12);
+ Binary |= (Reg << 13);
+ return Binary;
+ }
+
+ unsigned getHiLo16ImmOpValue(const MachineInstr &MI, unsigned Op) const {
+ return 0;
+ }
+
+ uint32_t getAddrMode2OffsetOpValue(const MachineInstr &MI, unsigned OpIdx)
+ const { return 0;}
+ uint32_t getPostIdxRegOpValue(const MachineInstr &MI, unsigned OpIdx)
+ const { return 0;}
+ uint32_t getAddrMode3OffsetOpValue(const MachineInstr &MI, unsigned OpIdx)
+ const { return 0;}
+ uint32_t getAddrMode3OpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ uint32_t getAddrModeThumbSPOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ uint32_t getAddrModeISOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ uint32_t getAddrModePCOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ uint32_t getAddrMode5OpValue(const MachineInstr &MI, unsigned Op) const {
+ // {17-13} = reg
+ // {12} = (U)nsigned (add == '1', sub == '0')
+ // {11-0} = imm12
+ const MachineOperand &MO = MI.getOperand(Op);
+ const MachineOperand &MO1 = MI.getOperand(Op + 1);
+ if (!MO.isReg()) {
+ emitConstPoolAddress(MO.getIndex(), ARM::reloc_arm_cp_entry);
+ return 0;
+ }
+ unsigned Reg = II->getRegisterInfo().getEncodingValue(MO.getReg());
+ int32_t Imm12 = MO1.getImm();
+
+ // Special value for #-0
+ if (Imm12 == INT32_MIN)
+ Imm12 = 0;
+
+ // Immediate is always encoded as positive. The 'U' bit controls add vs
+ // sub.
+ bool isAdd = true;
+ if (Imm12 < 0) {
+ Imm12 = -Imm12;
+ isAdd = false;
+ }
+
+ uint32_t Binary = Imm12 & 0xfff;
+ if (isAdd)
+ Binary |= (1 << 12);
+ Binary |= (Reg << 13);
+ return Binary;
+ }
+ unsigned getNEONVcvtImm32OpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+
+ unsigned getRegisterListOpValue(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+
+ unsigned getShiftRight8Imm(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getShiftRight16Imm(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getShiftRight32Imm(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+ unsigned getShiftRight64Imm(const MachineInstr &MI, unsigned Op)
+ const { return 0; }
+
+ /// getMovi32Value - Return binary encoding of operand for movw/movt. If the
+ /// machine operand requires relocation, record the relocation and return
+ /// zero.
+ unsigned getMovi32Value(const MachineInstr &MI,const MachineOperand &MO,
+ unsigned Reloc);
+
+ /// getShiftOp - Return the shift opcode (bit[6:5]) of the immediate value.
+ ///
+ unsigned getShiftOp(unsigned Imm) const ;
+
+ /// Routines that handle operands which add machine relocations which are
+ /// fixed up by the relocation stage.
+ void emitGlobalAddress(const GlobalValue *GV, unsigned Reloc,
+ bool MayNeedFarStub, bool Indirect,
+ intptr_t ACPV = 0) const;
+ void emitExternalSymbolAddress(const char *ES, unsigned Reloc) const;
+ void emitConstPoolAddress(unsigned CPI, unsigned Reloc) const;
+ void emitJumpTableAddress(unsigned JTIndex, unsigned Reloc) const;
+ void emitMachineBasicBlock(MachineBasicBlock *BB, unsigned Reloc,
+ intptr_t JTBase = 0) const;
+ unsigned encodeVFPRd(const MachineInstr &MI, unsigned OpIdx) const;
+ unsigned encodeVFPRn(const MachineInstr &MI, unsigned OpIdx) const;
+ unsigned encodeVFPRm(const MachineInstr &MI, unsigned OpIdx) const;
+ unsigned encodeNEONRd(const MachineInstr &MI, unsigned OpIdx) const;
+ unsigned encodeNEONRn(const MachineInstr &MI, unsigned OpIdx) const;
+ unsigned encodeNEONRm(const MachineInstr &MI, unsigned OpIdx) const;
+ };
+}
+
+char ARMCodeEmitter::ID = 0;
+
+/// createARMJITCodeEmitterPass - Return a pass that emits the collected ARM
+/// code to the specified MCE object.
+FunctionPass *llvm::createARMJITCodeEmitterPass(ARMBaseTargetMachine &TM,
+ JITCodeEmitter &JCE) {
+ return new ARMCodeEmitter(TM, JCE);
+}
+
+bool ARMCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
+ TargetMachine &Target = const_cast<TargetMachine&>(MF.getTarget());
+
+ assert((Target.getRelocationModel() != Reloc::Default ||
+ Target.getRelocationModel() != Reloc::Static) &&
+ "JIT relocation model must be set to static or default!");
+ // Initialize the subtarget first so we can grab all of the
+ // subtarget dependent variables from there.
+ Subtarget = &TM.getSubtarget<ARMSubtarget>();
+ JTI = static_cast<ARMJITInfo *>(Target.getSubtargetImpl()->getJITInfo());
+ II = static_cast<const ARMBaseInstrInfo *>(Subtarget->getInstrInfo());
+ TD = Target.getSubtargetImpl()->getDataLayout();
+
+ MCPEs = &MF.getConstantPool()->getConstants();
+ MJTEs = nullptr;
+ if (MF.getJumpTableInfo()) MJTEs = &MF.getJumpTableInfo()->getJumpTables();
+ IsPIC = TM.getRelocationModel() == Reloc::PIC_;
+ IsThumb = MF.getInfo<ARMFunctionInfo>()->isThumbFunction();
+ JTI->Initialize(MF, IsPIC);
+ MMI = &getAnalysis<MachineModuleInfo>();
+ MCE.setModuleInfo(MMI);
+
+ do {
+ DEBUG(errs() << "JITTing function '"
+ << MF.getName() << "'\n");
+ MCE.startFunction(MF);
+ for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
+ MBB != E; ++MBB) {
+ MCE.StartMachineBasicBlock(MBB);
+ for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
+ I != E; ++I)
+ emitInstruction(*I);
+ }
+ } while (MCE.finishFunction(MF));
+
+ return false;
+}
+
+/// getShiftOp - Return the shift opcode (bit[6:5]) of the immediate value.
+///
+unsigned ARMCodeEmitter::getShiftOp(unsigned Imm) const {
+ switch (ARM_AM::getAM2ShiftOpc(Imm)) {
+ default: llvm_unreachable("Unknown shift opc!");
+ case ARM_AM::asr: return 2;
+ case ARM_AM::lsl: return 0;
+ case ARM_AM::lsr: return 1;
+ case ARM_AM::ror:
+ case ARM_AM::rrx: return 3;
+ }
+}
+
+/// getMovi32Value - Return binary encoding of operand for movw/movt. If the
+/// machine operand requires relocation, record the relocation and return zero.
+unsigned ARMCodeEmitter::getMovi32Value(const MachineInstr &MI,
+ const MachineOperand &MO,
+ unsigned Reloc) {
+ assert(((Reloc == ARM::reloc_arm_movt) || (Reloc == ARM::reloc_arm_movw))
+ && "Relocation to this function should be for movt or movw");
+
+ if (MO.isImm())
+ return static_cast<unsigned>(MO.getImm());
+ else if (MO.isGlobal())
+ emitGlobalAddress(MO.getGlobal(), Reloc, true, false);
+ else if (MO.isSymbol())
+ emitExternalSymbolAddress(MO.getSymbolName(), Reloc);
+ else if (MO.isMBB())
+ emitMachineBasicBlock(MO.getMBB(), Reloc);
+ else {
+#ifndef NDEBUG
+ errs() << MO;
+#endif
+ llvm_unreachable("Unsupported operand type for movw/movt");
+ }
+ return 0;
+}
+
+/// getMachineOpValue - Return binary encoding of operand. If the machine
+/// operand requires relocation, record the relocation and return zero.
+unsigned ARMCodeEmitter::getMachineOpValue(const MachineInstr &MI,
+ const MachineOperand &MO) const {
+ if (MO.isReg())
+ return II->getRegisterInfo().getEncodingValue(MO.getReg());
+ else if (MO.isImm())
+ return static_cast<unsigned>(MO.getImm());
+ else if (MO.isGlobal())
+ emitGlobalAddress(MO.getGlobal(), ARM::reloc_arm_branch, true, false);
+ else if (MO.isSymbol())
+ emitExternalSymbolAddress(MO.getSymbolName(), ARM::reloc_arm_branch);
+ else if (MO.isCPI()) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ // For VFP load, the immediate offset is multiplied by 4.
+ unsigned Reloc = ((MCID.TSFlags & ARMII::FormMask) == ARMII::VFPLdStFrm)
+ ? ARM::reloc_arm_vfp_cp_entry : ARM::reloc_arm_cp_entry;
+ emitConstPoolAddress(MO.getIndex(), Reloc);
+ } else if (MO.isJTI())
+ emitJumpTableAddress(MO.getIndex(), ARM::reloc_arm_relative);
+ else if (MO.isMBB())
+ emitMachineBasicBlock(MO.getMBB(), ARM::reloc_arm_branch);
+ else
+ llvm_unreachable("Unable to encode MachineOperand!");
+ return 0;
+}
+
+/// emitGlobalAddress - Emit the specified address to the code stream.
+///
+void ARMCodeEmitter::emitGlobalAddress(const GlobalValue *GV, unsigned Reloc,
+ bool MayNeedFarStub, bool Indirect,
+ intptr_t ACPV) const {
+ MachineRelocation MR = Indirect
+ ? MachineRelocation::getIndirectSymbol(MCE.getCurrentPCOffset(), Reloc,
+ const_cast<GlobalValue *>(GV),
+ ACPV, MayNeedFarStub)
+ : MachineRelocation::getGV(MCE.getCurrentPCOffset(), Reloc,
+ const_cast<GlobalValue *>(GV), ACPV,
+ MayNeedFarStub);
+ MCE.addRelocation(MR);
+}
+
+/// emitExternalSymbolAddress - Arrange for the address of an external symbol to
+/// be emitted to the current location in the function, and allow it to be PC
+/// relative.
+void ARMCodeEmitter::
+emitExternalSymbolAddress(const char *ES, unsigned Reloc) const {
+ MCE.addRelocation(MachineRelocation::getExtSym(MCE.getCurrentPCOffset(),
+ Reloc, ES));
+}
+
+/// emitConstPoolAddress - Arrange for the address of an constant pool
+/// to be emitted to the current location in the function, and allow it to be PC
+/// relative.
+void ARMCodeEmitter::emitConstPoolAddress(unsigned CPI, unsigned Reloc) const {
+ // Tell JIT emitter we'll resolve the address.
+ MCE.addRelocation(MachineRelocation::getConstPool(MCE.getCurrentPCOffset(),
+ Reloc, CPI, 0, true));
+}
+
+/// emitJumpTableAddress - Arrange for the address of a jump table to
+/// be emitted to the current location in the function, and allow it to be PC
+/// relative.
+void ARMCodeEmitter::
+emitJumpTableAddress(unsigned JTIndex, unsigned Reloc) const {
+ MCE.addRelocation(MachineRelocation::getJumpTable(MCE.getCurrentPCOffset(),
+ Reloc, JTIndex, 0, true));
+}
+
+/// emitMachineBasicBlock - Emit the specified address basic block.
+void ARMCodeEmitter::emitMachineBasicBlock(MachineBasicBlock *BB,
+ unsigned Reloc,
+ intptr_t JTBase) const {
+ MCE.addRelocation(MachineRelocation::getBB(MCE.getCurrentPCOffset(),
+ Reloc, BB, JTBase));
+}
+
+void ARMCodeEmitter::emitWordLE(unsigned Binary) {
+ DEBUG(errs() << " 0x";
+ errs().write_hex(Binary) << "\n");
+ MCE.emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitDWordLE(uint64_t Binary) {
+ DEBUG(errs() << " 0x";
+ errs().write_hex(Binary) << "\n");
+ MCE.emitDWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitInstruction(const MachineInstr &MI) {
+ DEBUG(errs() << "JIT: " << (void*)MCE.getCurrentPCValue() << ":\t" << MI);
+
+ MCE.processDebugLoc(MI.getDebugLoc(), true);
+
+ ++NumEmitted; // Keep track of the # of mi's emitted
+ switch (MI.getDesc().TSFlags & ARMII::FormMask) {
+ default: {
+ llvm_unreachable("Unhandled instruction encoding format!");
+ }
+ case ARMII::MiscFrm:
+ if (MI.getOpcode() == ARM::LEApcrelJT) {
+ // Materialize jumptable address.
+ emitLEApcrelJTInstruction(MI);
+ break;
+ }
+ llvm_unreachable("Unhandled instruction encoding!");
+ case ARMII::Pseudo:
+ emitPseudoInstruction(MI);
+ break;
+ case ARMII::DPFrm:
+ case ARMII::DPSoRegFrm:
+ emitDataProcessingInstruction(MI);
+ break;
+ case ARMII::LdFrm:
+ case ARMII::StFrm:
+ emitLoadStoreInstruction(MI);
+ break;
+ case ARMII::LdMiscFrm:
+ case ARMII::StMiscFrm:
+ emitMiscLoadStoreInstruction(MI);
+ break;
+ case ARMII::LdStMulFrm:
+ emitLoadStoreMultipleInstruction(MI);
+ break;
+ case ARMII::MulFrm:
+ emitMulFrmInstruction(MI);
+ break;
+ case ARMII::ExtFrm:
+ emitExtendInstruction(MI);
+ break;
+ case ARMII::ArithMiscFrm:
+ emitMiscArithInstruction(MI);
+ break;
+ case ARMII::SatFrm:
+ emitSaturateInstruction(MI);
+ break;
+ case ARMII::BrFrm:
+ emitBranchInstruction(MI);
+ break;
+ case ARMII::BrMiscFrm:
+ emitMiscBranchInstruction(MI);
+ break;
+ // VFP instructions.
+ case ARMII::VFPUnaryFrm:
+ case ARMII::VFPBinaryFrm:
+ emitVFPArithInstruction(MI);
+ break;
+ case ARMII::VFPConv1Frm:
+ case ARMII::VFPConv2Frm:
+ case ARMII::VFPConv3Frm:
+ case ARMII::VFPConv4Frm:
+ case ARMII::VFPConv5Frm:
+ emitVFPConversionInstruction(MI);
+ break;
+ case ARMII::VFPLdStFrm:
+ emitVFPLoadStoreInstruction(MI);
+ break;
+ case ARMII::VFPLdStMulFrm:
+ emitVFPLoadStoreMultipleInstruction(MI);
+ break;
+
+ // NEON instructions.
+ case ARMII::NGetLnFrm:
+ case ARMII::NSetLnFrm:
+ emitNEONLaneInstruction(MI);
+ break;
+ case ARMII::NDupFrm:
+ emitNEONDupInstruction(MI);
+ break;
+ case ARMII::N1RegModImmFrm:
+ emitNEON1RegModImmInstruction(MI);
+ break;
+ case ARMII::N2RegFrm:
+ emitNEON2RegInstruction(MI);
+ break;
+ case ARMII::N3RegFrm:
+ emitNEON3RegInstruction(MI);
+ break;
+ }
+ MCE.processDebugLoc(MI.getDebugLoc(), false);
+}
+
+void ARMCodeEmitter::emitConstPoolInstruction(const MachineInstr &MI) {
+ unsigned CPI = MI.getOperand(0).getImm(); // CP instruction index.
+ unsigned CPIndex = MI.getOperand(1).getIndex(); // Actual cp entry index.
+ const MachineConstantPoolEntry &MCPE = (*MCPEs)[CPIndex];
+
+ // Remember the CONSTPOOL_ENTRY address for later relocation.
+ JTI->addConstantPoolEntryAddr(CPI, MCE.getCurrentPCValue());
+
+ // Emit constpool island entry. In most cases, the actual values will be
+ // resolved and relocated after code emission.
+ if (MCPE.isMachineConstantPoolEntry()) {
+ ARMConstantPoolValue *ACPV =
+ static_cast<ARMConstantPoolValue*>(MCPE.Val.MachineCPVal);
+
+ DEBUG(errs() << " ** ARM constant pool #" << CPI << " @ "
+ << (void*)MCE.getCurrentPCValue() << " " << *ACPV << '\n');
+
+ assert(ACPV->isGlobalValue() && "unsupported constant pool value");
+ const GlobalValue *GV = cast<ARMConstantPoolConstant>(ACPV)->getGV();
+ if (GV) {
+ Reloc::Model RelocM = TM.getRelocationModel();
+ emitGlobalAddress(GV, ARM::reloc_arm_machine_cp_entry,
+ isa<Function>(GV),
+ Subtarget->GVIsIndirectSymbol(GV, RelocM),
+ (intptr_t)ACPV);
+ } else {
+ const char *Sym = cast<ARMConstantPoolSymbol>(ACPV)->getSymbol();
+ emitExternalSymbolAddress(Sym, ARM::reloc_arm_absolute);
+ }
+ emitWordLE(0);
+ } else {
+ const Constant *CV = MCPE.Val.ConstVal;
+
+ DEBUG({
+ errs() << " ** Constant pool #" << CPI << " @ "
+ << (void*)MCE.getCurrentPCValue() << " ";
+ if (const Function *F = dyn_cast<Function>(CV))
+ errs() << F->getName();
+ else
+ errs() << *CV;
+ errs() << '\n';
+ });
+
+ if (const GlobalValue *GV = dyn_cast<GlobalValue>(CV)) {
+ emitGlobalAddress(GV, ARM::reloc_arm_absolute, isa<Function>(GV), false);
+ emitWordLE(0);
+ } else if (const ConstantInt *CI = dyn_cast<ConstantInt>(CV)) {
+ uint32_t Val = uint32_t(*CI->getValue().getRawData());
+ emitWordLE(Val);
+ } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CV)) {
+ if (CFP->getType()->isFloatTy())
+ emitWordLE(CFP->getValueAPF().bitcastToAPInt().getZExtValue());
+ else if (CFP->getType()->isDoubleTy())
+ emitDWordLE(CFP->getValueAPF().bitcastToAPInt().getZExtValue());
+ else {
+ llvm_unreachable("Unable to handle this constantpool entry!");
+ }
+ } else {
+ llvm_unreachable("Unable to handle this constantpool entry!");
+ }
+ }
+}
+
+void ARMCodeEmitter::emitMOVi32immInstruction(const MachineInstr &MI) {
+ const MachineOperand &MO0 = MI.getOperand(0);
+ const MachineOperand &MO1 = MI.getOperand(1);
+
+ // Emit the 'movw' instruction.
+ unsigned Binary = 0x30 << 20; // mov: Insts{27-20} = 0b00110000
+
+ unsigned Lo16 = getMovi32Value(MI, MO1, ARM::reloc_arm_movw) & 0xFFFF;
+
+ // Set the conditional execution predicate.
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode Rd.
+ Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRdShift;
+
+ // Encode imm16 as imm4:imm12
+ Binary |= Lo16 & 0xFFF; // Insts{11-0} = imm12
+ Binary |= ((Lo16 >> 12) & 0xF) << 16; // Insts{19-16} = imm4
+ emitWordLE(Binary);
+
+ unsigned Hi16 = getMovi32Value(MI, MO1, ARM::reloc_arm_movt) >> 16;
+ // Emit the 'movt' instruction.
+ Binary = 0x34 << 20; // movt: Insts{27-20} = 0b00110100
+
+ // Set the conditional execution predicate.
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode Rd.
+ Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRdShift;
+
+ // Encode imm16 as imm4:imm1, same as movw above.
+ Binary |= Hi16 & 0xFFF;
+ Binary |= ((Hi16 >> 12) & 0xF) << 16;
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitMOVi2piecesInstruction(const MachineInstr &MI) {
+ const MachineOperand &MO0 = MI.getOperand(0);
+ const MachineOperand &MO1 = MI.getOperand(1);
+ assert(MO1.isImm() && ARM_AM::isSOImmTwoPartVal(MO1.getImm()) &&
+ "Not a valid so_imm value!");
+ unsigned V1 = ARM_AM::getSOImmTwoPartFirst(MO1.getImm());
+ unsigned V2 = ARM_AM::getSOImmTwoPartSecond(MO1.getImm());
+
+ // Emit the 'mov' instruction.
+ unsigned Binary = 0xd << 21; // mov: Insts{24-21} = 0b1101
+
+ // Set the conditional execution predicate.
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode Rd.
+ Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRdShift;
+
+ // Encode so_imm.
+ // Set bit I(25) to identify this is the immediate form of <shifter_op>
+ Binary |= 1 << ARMII::I_BitShift;
+ Binary |= getMachineSoImmOpValue(V1);
+ emitWordLE(Binary);
+
+ // Now the 'orr' instruction.
+ Binary = 0xc << 21; // orr: Insts{24-21} = 0b1100
+
+ // Set the conditional execution predicate.
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode Rd.
+ Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRdShift;
+
+ // Encode Rn.
+ Binary |= getMachineOpValue(MI, MO0) << ARMII::RegRnShift;
+
+ // Encode so_imm.
+ // Set bit I(25) to identify this is the immediate form of <shifter_op>
+ Binary |= 1 << ARMII::I_BitShift;
+ Binary |= getMachineSoImmOpValue(V2);
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitLEApcrelJTInstruction(const MachineInstr &MI) {
+ // It's basically add r, pc, (LJTI - $+8)
+
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Emit the 'add' instruction.
+ unsigned Binary = 0x4 << 21; // add: Insts{24-21} = 0b0100
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode S bit if MI modifies CPSR.
+ Binary |= getAddrModeSBit(MI, MCID);
+
+ // Encode Rd.
+ Binary |= getMachineOpValue(MI, 0) << ARMII::RegRdShift;
+
+ // Encode Rn which is PC.
+ Binary |= II->getRegisterInfo().getEncodingValue(ARM::PC) << ARMII::RegRnShift;
+
+ // Encode the displacement.
+ Binary |= 1 << ARMII::I_BitShift;
+ emitJumpTableAddress(MI.getOperand(1).getIndex(), ARM::reloc_arm_jt_base);
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitPseudoMoveInstruction(const MachineInstr &MI) {
+ unsigned Opcode = MI.getDesc().Opcode;
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode S bit if MI modifies CPSR.
+ if (Opcode == ARM::MOVsrl_flag || Opcode == ARM::MOVsra_flag)
+ Binary |= 1 << ARMII::S_BitShift;
+
+ // Encode register def if there is one.
+ Binary |= getMachineOpValue(MI, 0) << ARMII::RegRdShift;
+
+ // Encode the shift operation.
+ switch (Opcode) {
+ default: break;
+ case ARM::RRX:
+ // rrx
+ Binary |= 0x6 << 4;
+ break;
+ case ARM::MOVsrl_flag:
+ // lsr #1
+ Binary |= (0x2 << 4) | (1 << 7);
+ break;
+ case ARM::MOVsra_flag:
+ // asr #1
+ Binary |= (0x4 << 4) | (1 << 7);
+ break;
+ }
+
+ // Encode register Rm.
+ Binary |= getMachineOpValue(MI, 1);
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::addPCLabel(unsigned LabelID) {
+ DEBUG(errs() << " ** LPC" << LabelID << " @ "
+ << (void*)MCE.getCurrentPCValue() << '\n');
+ JTI->addPCLabelAddr(LabelID, MCE.getCurrentPCValue());
+}
+
+void ARMCodeEmitter::emitPseudoInstruction(const MachineInstr &MI) {
+ unsigned Opcode = MI.getDesc().Opcode;
+ switch (Opcode) {
+ default:
+ llvm_unreachable("ARMCodeEmitter::emitPseudoInstruction");
+ case ARM::BX_CALL:
+ case ARM::BMOVPCRX_CALL: {
+ // First emit mov lr, pc
+ unsigned Binary = 0x01a0e00f;
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+ emitWordLE(Binary);
+
+ // and then emit the branch.
+ emitMiscBranchInstruction(MI);
+ break;
+ }
+ case TargetOpcode::INLINEASM: {
+ // We allow inline assembler nodes with empty bodies - they can
+ // implicitly define registers, which is ok for JIT.
+ if (MI.getOperand(0).getSymbolName()[0]) {
+ report_fatal_error("JIT does not support inline asm!");
+ }
+ break;
+ }
+ case TargetOpcode::CFI_INSTRUCTION:
+ break;
+ case TargetOpcode::EH_LABEL:
+ MCE.emitLabel(MI.getOperand(0).getMCSymbol());
+ break;
+ case TargetOpcode::IMPLICIT_DEF:
+ case TargetOpcode::KILL:
+ // Do nothing.
+ break;
+ case ARM::CONSTPOOL_ENTRY:
+ emitConstPoolInstruction(MI);
+ break;
+ case ARM::PICADD: {
+ // Remember of the address of the PC label for relocation later.
+ addPCLabel(MI.getOperand(2).getImm());
+ // PICADD is just an add instruction that implicitly read pc.
+ emitDataProcessingInstruction(MI, 0, ARM::PC);
+ break;
+ }
+ case ARM::PICLDR:
+ case ARM::PICLDRB:
+ case ARM::PICSTR:
+ case ARM::PICSTRB: {
+ // Remember of the address of the PC label for relocation later.
+ addPCLabel(MI.getOperand(2).getImm());
+ // These are just load / store instructions that implicitly read pc.
+ emitLoadStoreInstruction(MI, 0, ARM::PC);
+ break;
+ }
+ case ARM::PICLDRH:
+ case ARM::PICLDRSH:
+ case ARM::PICLDRSB:
+ case ARM::PICSTRH: {
+ // Remember of the address of the PC label for relocation later.
+ addPCLabel(MI.getOperand(2).getImm());
+ // These are just load / store instructions that implicitly read pc.
+ emitMiscLoadStoreInstruction(MI, ARM::PC);
+ break;
+ }
+
+ case ARM::MOVi32imm:
+ // Two instructions to materialize a constant.
+ if (Subtarget->hasV6T2Ops())
+ emitMOVi32immInstruction(MI);
+ else
+ emitMOVi2piecesInstruction(MI);
+ break;
+
+ case ARM::LEApcrelJT:
+ // Materialize jumptable address.
+ emitLEApcrelJTInstruction(MI);
+ break;
+ case ARM::RRX:
+ case ARM::MOVsrl_flag:
+ case ARM::MOVsra_flag:
+ emitPseudoMoveInstruction(MI);
+ break;
+ }
+}
+
+unsigned ARMCodeEmitter::getMachineSoRegOpValue(const MachineInstr &MI,
+ const MCInstrDesc &MCID,
+ const MachineOperand &MO,
+ unsigned OpIdx) {
+ unsigned Binary = getMachineOpValue(MI, MO);
+
+ const MachineOperand &MO1 = MI.getOperand(OpIdx + 1);
+ const MachineOperand &MO2 = MI.getOperand(OpIdx + 2);
+ ARM_AM::ShiftOpc SOpc = ARM_AM::getSORegShOp(MO2.getImm());
+
+ // Encode the shift opcode.
+ unsigned SBits = 0;
+ unsigned Rs = MO1.getReg();
+ if (Rs) {
+ // Set shift operand (bit[7:4]).
+ // LSL - 0001
+ // LSR - 0011
+ // ASR - 0101
+ // ROR - 0111
+ // RRX - 0110 and bit[11:8] clear.
+ switch (SOpc) {
+ default: llvm_unreachable("Unknown shift opc!");
+ case ARM_AM::lsl: SBits = 0x1; break;
+ case ARM_AM::lsr: SBits = 0x3; break;
+ case ARM_AM::asr: SBits = 0x5; break;
+ case ARM_AM::ror: SBits = 0x7; break;
+ case ARM_AM::rrx: SBits = 0x6; break;
+ }
+ } else {
+ // Set shift operand (bit[6:4]).
+ // LSL - 000
+ // LSR - 010
+ // ASR - 100
+ // ROR - 110
+ switch (SOpc) {
+ default: llvm_unreachable("Unknown shift opc!");
+ case ARM_AM::lsl: SBits = 0x0; break;
+ case ARM_AM::lsr: SBits = 0x2; break;
+ case ARM_AM::asr: SBits = 0x4; break;
+ case ARM_AM::ror: SBits = 0x6; break;
+ }
+ }
+ Binary |= SBits << 4;
+ if (SOpc == ARM_AM::rrx)
+ return Binary;
+
+ // Encode the shift operation Rs or shift_imm (except rrx).
+ if (Rs) {
+ // Encode Rs bit[11:8].
+ assert(ARM_AM::getSORegOffset(MO2.getImm()) == 0);
+ return Binary | (II->getRegisterInfo().getEncodingValue(Rs) << ARMII::RegRsShift);
+ }
+
+ // Encode shift_imm bit[11:7].
+ return Binary | ARM_AM::getSORegOffset(MO2.getImm()) << 7;
+}
+
+unsigned ARMCodeEmitter::getMachineSoImmOpValue(unsigned SoImm) {
+ int SoImmVal = ARM_AM::getSOImmVal(SoImm);
+ assert(SoImmVal != -1 && "Not a valid so_imm value!");
+
+ // Encode rotate_imm.
+ unsigned Binary = (ARM_AM::getSOImmValRot((unsigned)SoImmVal) >> 1)
+ << ARMII::SoRotImmShift;
+
+ // Encode immed_8.
+ Binary |= ARM_AM::getSOImmValImm((unsigned)SoImmVal);
+ return Binary;
+}
+
+unsigned ARMCodeEmitter::getAddrModeSBit(const MachineInstr &MI,
+ const MCInstrDesc &MCID) const {
+ for (unsigned i = MI.getNumOperands(), e = MCID.getNumOperands(); i >= e;--i){
+ const MachineOperand &MO = MI.getOperand(i-1);
+ if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR)
+ return 1 << ARMII::S_BitShift;
+ }
+ return 0;
+}
+
+void ARMCodeEmitter::emitDataProcessingInstruction(const MachineInstr &MI,
+ unsigned ImplicitRd,
+ unsigned ImplicitRn) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode S bit if MI modifies CPSR.
+ Binary |= getAddrModeSBit(MI, MCID);
+
+ // Encode register def if there is one.
+ unsigned NumDefs = MCID.getNumDefs();
+ unsigned OpIdx = 0;
+ if (NumDefs)
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
+ else if (ImplicitRd)
+ // Special handling for implicit use (e.g. PC).
+ Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRd) << ARMII::RegRdShift);
+
+ if (MCID.Opcode == ARM::MOVi16) {
+ // Get immediate from MI.
+ unsigned Lo16 = getMovi32Value(MI, MI.getOperand(OpIdx),
+ ARM::reloc_arm_movw);
+ // Encode imm which is the same as in emitMOVi32immInstruction().
+ Binary |= Lo16 & 0xFFF;
+ Binary |= ((Lo16 >> 12) & 0xF) << 16;
+ emitWordLE(Binary);
+ return;
+ } else if(MCID.Opcode == ARM::MOVTi16) {
+ unsigned Hi16 = (getMovi32Value(MI, MI.getOperand(OpIdx),
+ ARM::reloc_arm_movt) >> 16);
+ Binary |= Hi16 & 0xFFF;
+ Binary |= ((Hi16 >> 12) & 0xF) << 16;
+ emitWordLE(Binary);
+ return;
+ } else if ((MCID.Opcode == ARM::BFC) || (MCID.Opcode == ARM::BFI)) {
+ uint32_t v = ~MI.getOperand(2).getImm();
+ int32_t lsb = countTrailingZeros(v);
+ int32_t msb = (32 - countLeadingZeros(v)) - 1;
+ // Instr{20-16} = msb, Instr{11-7} = lsb
+ Binary |= (msb & 0x1F) << 16;
+ Binary |= (lsb & 0x1F) << 7;
+ emitWordLE(Binary);
+ return;
+ } else if ((MCID.Opcode == ARM::UBFX) || (MCID.Opcode == ARM::SBFX)) {
+ // Encode Rn in Instr{0-3}
+ Binary |= getMachineOpValue(MI, OpIdx++);
+
+ uint32_t lsb = MI.getOperand(OpIdx++).getImm();
+ uint32_t widthm1 = MI.getOperand(OpIdx++).getImm() - 1;
+
+ // Instr{20-16} = widthm1, Instr{11-7} = lsb
+ Binary |= (widthm1 & 0x1F) << 16;
+ Binary |= (lsb & 0x1F) << 7;
+ emitWordLE(Binary);
+ return;
+ }
+
+ // If this is a two-address operand, skip it. e.g. MOVCCr operand 1.
+ if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
+ ++OpIdx;
+
+ // Encode first non-shifter register operand if there is one.
+ bool isUnary = MCID.TSFlags & ARMII::UnaryDP;
+ if (!isUnary) {
+ if (ImplicitRn)
+ // Special handling for implicit use (e.g. PC).
+ Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRn) << ARMII::RegRnShift);
+ else {
+ Binary |= getMachineOpValue(MI, OpIdx) << ARMII::RegRnShift;
+ ++OpIdx;
+ }
+ }
+
+ // Encode shifter operand.
+ const MachineOperand &MO = MI.getOperand(OpIdx);
+ if ((MCID.TSFlags & ARMII::FormMask) == ARMII::DPSoRegFrm) {
+ // Encode SoReg.
+ emitWordLE(Binary | getMachineSoRegOpValue(MI, MCID, MO, OpIdx));
+ return;
+ }
+
+ if (MO.isReg()) {
+ // Encode register Rm.
+ emitWordLE(Binary | II->getRegisterInfo().getEncodingValue(MO.getReg()));
+ return;
+ }
+
+ // Encode so_imm.
+ Binary |= getMachineSoImmOpValue((unsigned)MO.getImm());
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitLoadStoreInstruction(const MachineInstr &MI,
+ unsigned ImplicitRd,
+ unsigned ImplicitRn) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ unsigned Form = MCID.TSFlags & ARMII::FormMask;
+ bool IsPrePost = (MCID.TSFlags & ARMII::IndexModeMask) != 0;
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // If this is an LDRi12, STRi12 or LDRcp, nothing more needs be done.
+ if (MI.getOpcode() == ARM::LDRi12 || MI.getOpcode() == ARM::LDRcp ||
+ MI.getOpcode() == ARM::STRi12) {
+ emitWordLE(Binary);
+ return;
+ }
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ unsigned OpIdx = 0;
+
+ // Operand 0 of a pre- and post-indexed store is the address base
+ // writeback. Skip it.
+ bool Skipped = false;
+ if (IsPrePost && Form == ARMII::StFrm) {
+ ++OpIdx;
+ Skipped = true;
+ }
+
+ // Set first operand
+ if (ImplicitRd)
+ // Special handling for implicit use (e.g. PC).
+ Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRd) << ARMII::RegRdShift);
+ else
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
+
+ // Set second operand
+ if (ImplicitRn)
+ // Special handling for implicit use (e.g. PC).
+ Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRn) << ARMII::RegRnShift);
+ else
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
+
+ // If this is a two-address operand, skip it. e.g. LDR_PRE.
+ if (!Skipped && MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
+ ++OpIdx;
+
+ const MachineOperand &MO2 = MI.getOperand(OpIdx);
+ unsigned AM2Opc = (ImplicitRn == ARM::PC)
+ ? 0 : MI.getOperand(OpIdx+1).getImm();
+
+ // Set bit U(23) according to sign of immed value (positive or negative).
+ Binary |= ((ARM_AM::getAM2Op(AM2Opc) == ARM_AM::add ? 1 : 0) <<
+ ARMII::U_BitShift);
+ if (!MO2.getReg()) { // is immediate
+ if (ARM_AM::getAM2Offset(AM2Opc))
+ // Set the value of offset_12 field
+ Binary |= ARM_AM::getAM2Offset(AM2Opc);
+ emitWordLE(Binary);
+ return;
+ }
+
+ // Set bit I(25), because this is not in immediate encoding.
+ Binary |= 1 << ARMII::I_BitShift;
+ assert(TargetRegisterInfo::isPhysicalRegister(MO2.getReg()));
+ // Set bit[3:0] to the corresponding Rm register
+ Binary |= II->getRegisterInfo().getEncodingValue(MO2.getReg());
+
+ // If this instr is in scaled register offset/index instruction, set
+ // shift_immed(bit[11:7]) and shift(bit[6:5]) fields.
+ if (unsigned ShImm = ARM_AM::getAM2Offset(AM2Opc)) {
+ Binary |= getShiftOp(AM2Opc) << ARMII::ShiftImmShift; // shift
+ Binary |= ShImm << ARMII::ShiftShift; // shift_immed
+ }
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitMiscLoadStoreInstruction(const MachineInstr &MI,
+ unsigned ImplicitRn) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ unsigned Form = MCID.TSFlags & ARMII::FormMask;
+ bool IsPrePost = (MCID.TSFlags & ARMII::IndexModeMask) != 0;
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ unsigned OpIdx = 0;
+
+ // Operand 0 of a pre- and post-indexed store is the address base
+ // writeback. Skip it.
+ bool Skipped = false;
+ if (IsPrePost && Form == ARMII::StMiscFrm) {
+ ++OpIdx;
+ Skipped = true;
+ }
+
+ // Set first operand
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
+
+ // Skip LDRD and STRD's second operand.
+ if (MCID.Opcode == ARM::LDRD || MCID.Opcode == ARM::STRD)
+ ++OpIdx;
+
+ // Set second operand
+ if (ImplicitRn)
+ // Special handling for implicit use (e.g. PC).
+ Binary |= (II->getRegisterInfo().getEncodingValue(ImplicitRn) << ARMII::RegRnShift);
+ else
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
+
+ // If this is a two-address operand, skip it. e.g. LDRH_POST.
+ if (!Skipped && MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
+ ++OpIdx;
+
+ const MachineOperand &MO2 = MI.getOperand(OpIdx);
+ unsigned AM3Opc = (ImplicitRn == ARM::PC)
+ ? 0 : MI.getOperand(OpIdx+1).getImm();
+
+ // Set bit U(23) according to sign of immed value (positive or negative)
+ Binary |= ((ARM_AM::getAM3Op(AM3Opc) == ARM_AM::add ? 1 : 0) <<
+ ARMII::U_BitShift);
+
+ // If this instr is in register offset/index encoding, set bit[3:0]
+ // to the corresponding Rm register.
+ if (MO2.getReg()) {
+ Binary |= II->getRegisterInfo().getEncodingValue(MO2.getReg());
+ emitWordLE(Binary);
+ return;
+ }
+
+ // This instr is in immediate offset/index encoding, set bit 22 to 1.
+ Binary |= 1 << ARMII::AM3_I_BitShift;
+ if (unsigned ImmOffs = ARM_AM::getAM3Offset(AM3Opc)) {
+ // Set operands
+ Binary |= (ImmOffs >> 4) << ARMII::ImmHiShift; // immedH
+ Binary |= (ImmOffs & 0xF); // immedL
+ }
+
+ emitWordLE(Binary);
+}
+
+static unsigned getAddrModeUPBits(unsigned Mode) {
+ unsigned Binary = 0;
+
+ // Set addressing mode by modifying bits U(23) and P(24)
+ // IA - Increment after - bit U = 1 and bit P = 0
+ // IB - Increment before - bit U = 1 and bit P = 1
+ // DA - Decrement after - bit U = 0 and bit P = 0
+ // DB - Decrement before - bit U = 0 and bit P = 1
+ switch (Mode) {
+ default: llvm_unreachable("Unknown addressing sub-mode!");
+ case ARM_AM::da: break;
+ case ARM_AM::db: Binary |= 0x1 << ARMII::P_BitShift; break;
+ case ARM_AM::ia: Binary |= 0x1 << ARMII::U_BitShift; break;
+ case ARM_AM::ib: Binary |= 0x3 << ARMII::U_BitShift; break;
+ }
+
+ return Binary;
+}
+
+void ARMCodeEmitter::emitLoadStoreMultipleInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ bool IsUpdating = (MCID.TSFlags & ARMII::IndexModeMask) != 0;
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Skip operand 0 of an instruction with base register update.
+ unsigned OpIdx = 0;
+ if (IsUpdating)
+ ++OpIdx;
+
+ // Set base address operand
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
+
+ // Set addressing mode by modifying bits U(23) and P(24)
+ ARM_AM::AMSubMode Mode = ARM_AM::getLoadStoreMultipleSubMode(MI.getOpcode());
+ Binary |= getAddrModeUPBits(ARM_AM::getAM4SubMode(Mode));
+
+ // Set bit W(21)
+ if (IsUpdating)
+ Binary |= 0x1 << ARMII::W_BitShift;
+
+ // Set registers
+ for (unsigned i = OpIdx+2, e = MI.getNumOperands(); i != e; ++i) {
+ const MachineOperand &MO = MI.getOperand(i);
+ if (!MO.isReg() || MO.isImplicit())
+ break;
+ unsigned RegNum = II->getRegisterInfo().getEncodingValue(MO.getReg());
+ assert(TargetRegisterInfo::isPhysicalRegister(MO.getReg()) &&
+ RegNum < 16);
+ Binary |= 0x1 << RegNum;
+ }
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitMulFrmInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode S bit if MI modifies CPSR.
+ Binary |= getAddrModeSBit(MI, MCID);
+
+ // 32x32->64bit operations have two destination registers. The number
+ // of register definitions will tell us if that's what we're dealing with.
+ unsigned OpIdx = 0;
+ if (MCID.getNumDefs() == 2)
+ Binary |= getMachineOpValue (MI, OpIdx++) << ARMII::RegRdLoShift;
+
+ // Encode Rd
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdHiShift;
+
+ // Encode Rm
+ Binary |= getMachineOpValue(MI, OpIdx++);
+
+ // Encode Rs
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRsShift;
+
+ // Many multiple instructions (e.g. MLA) have three src operands. Encode
+ // it as Rn (for multiply, that's in the same offset as RdLo.
+ if (MCID.getNumOperands() > OpIdx &&
+ !MCID.OpInfo[OpIdx].isPredicate() &&
+ !MCID.OpInfo[OpIdx].isOptionalDef())
+ Binary |= getMachineOpValue(MI, OpIdx) << ARMII::RegRdLoShift;
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitExtendInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ unsigned OpIdx = 0;
+
+ // Encode Rd
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
+
+ const MachineOperand &MO1 = MI.getOperand(OpIdx++);
+ const MachineOperand &MO2 = MI.getOperand(OpIdx);
+ if (MO2.isReg()) {
+ // Two register operand form.
+ // Encode Rn.
+ Binary |= getMachineOpValue(MI, MO1) << ARMII::RegRnShift;
+
+ // Encode Rm.
+ Binary |= getMachineOpValue(MI, MO2);
+ ++OpIdx;
+ } else {
+ Binary |= getMachineOpValue(MI, MO1);
+ }
+
+ // Encode rot imm (0, 8, 16, or 24) if it has a rotate immediate operand.
+ if (MI.getOperand(OpIdx).isImm() &&
+ !MCID.OpInfo[OpIdx].isPredicate() &&
+ !MCID.OpInfo[OpIdx].isOptionalDef())
+ Binary |= (getMachineOpValue(MI, OpIdx) / 8) << ARMII::ExtRotImmShift;
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitMiscArithInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // PKH instructions are finished at this point
+ if (MCID.Opcode == ARM::PKHBT || MCID.Opcode == ARM::PKHTB) {
+ emitWordLE(Binary);
+ return;
+ }
+
+ unsigned OpIdx = 0;
+
+ // Encode Rd
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRdShift;
+
+ const MachineOperand &MO = MI.getOperand(OpIdx++);
+ if (OpIdx == MCID.getNumOperands() ||
+ MCID.OpInfo[OpIdx].isPredicate() ||
+ MCID.OpInfo[OpIdx].isOptionalDef()) {
+ // Encode Rm and it's done.
+ Binary |= getMachineOpValue(MI, MO);
+ emitWordLE(Binary);
+ return;
+ }
+
+ // Encode Rn.
+ Binary |= getMachineOpValue(MI, MO) << ARMII::RegRnShift;
+
+ // Encode Rm.
+ Binary |= getMachineOpValue(MI, OpIdx++);
+
+ // Encode shift_imm.
+ unsigned ShiftAmt = MI.getOperand(OpIdx).getImm();
+ if (MCID.Opcode == ARM::PKHTB) {
+ assert(ShiftAmt != 0 && "PKHTB shift_imm is 0!");
+ if (ShiftAmt == 32)
+ ShiftAmt = 0;
+ }
+ assert(ShiftAmt < 32 && "shift_imm range is 0 to 31!");
+ Binary |= ShiftAmt << ARMII::ShiftShift;
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitSaturateInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Part of binary is determined by TableGen.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Encode Rd
+ Binary |= getMachineOpValue(MI, 0) << ARMII::RegRdShift;
+
+ // Encode saturate bit position.
+ unsigned Pos = MI.getOperand(1).getImm();
+ if (MCID.Opcode == ARM::SSAT || MCID.Opcode == ARM::SSAT16)
+ Pos -= 1;
+ assert((Pos < 16 || (Pos < 32 &&
+ MCID.Opcode != ARM::SSAT16 &&
+ MCID.Opcode != ARM::USAT16)) &&
+ "saturate bit position out of range");
+ Binary |= Pos << 16;
+
+ // Encode Rm
+ Binary |= getMachineOpValue(MI, 2);
+
+ // Encode shift_imm.
+ if (MCID.getNumOperands() == 4) {
+ unsigned ShiftOp = MI.getOperand(3).getImm();
+ ARM_AM::ShiftOpc Opc = ARM_AM::getSORegShOp(ShiftOp);
+ if (Opc == ARM_AM::asr)
+ Binary |= (1 << 6);
+ unsigned ShiftAmt = MI.getOperand(3).getImm();
+ if (ShiftAmt == 32 && Opc == ARM_AM::asr)
+ ShiftAmt = 0;
+ assert(ShiftAmt < 32 && "shift_imm range is 0 to 31!");
+ Binary |= ShiftAmt << ARMII::ShiftShift;
+ }
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitBranchInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ if (MCID.Opcode == ARM::TPsoft) {
+ llvm_unreachable("ARM::TPsoft FIXME"); // FIXME
+ }
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Set signed_immed_24 field
+ Binary |= getMachineOpValue(MI, 0);
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitInlineJumpTable(unsigned JTIndex) {
+ // Remember the base address of the inline jump table.
+ uintptr_t JTBase = MCE.getCurrentPCValue();
+ JTI->addJumpTableBaseAddr(JTIndex, JTBase);
+ DEBUG(errs() << " ** Jump Table #" << JTIndex << " @ " << (void*)JTBase
+ << '\n');
+
+ // Now emit the jump table entries.
+ const std::vector<MachineBasicBlock*> &MBBs = (*MJTEs)[JTIndex].MBBs;
+ for (unsigned i = 0, e = MBBs.size(); i != e; ++i) {
+ if (IsPIC)
+ // DestBB address - JT base.
+ emitMachineBasicBlock(MBBs[i], ARM::reloc_arm_pic_jt, JTBase);
+ else
+ // Absolute DestBB address.
+ emitMachineBasicBlock(MBBs[i], ARM::reloc_arm_absolute);
+ emitWordLE(0);
+ }
+}
+
+void ARMCodeEmitter::emitMiscBranchInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Handle jump tables.
+ if (MCID.Opcode == ARM::BR_JTr || MCID.Opcode == ARM::BR_JTadd) {
+ // First emit a ldr pc, [] instruction.
+ emitDataProcessingInstruction(MI, ARM::PC);
+
+ // Then emit the inline jump table.
+ unsigned JTIndex =
+ (MCID.Opcode == ARM::BR_JTr)
+ ? MI.getOperand(1).getIndex() : MI.getOperand(2).getIndex();
+ emitInlineJumpTable(JTIndex);
+ return;
+ } else if (MCID.Opcode == ARM::BR_JTm) {
+ // First emit a ldr pc, [] instruction.
+ emitLoadStoreInstruction(MI, ARM::PC);
+
+ // Then emit the inline jump table.
+ emitInlineJumpTable(MI.getOperand(3).getIndex());
+ return;
+ }
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ if (MCID.Opcode == ARM::BX_RET || MCID.Opcode == ARM::MOVPCLR)
+ // The return register is LR.
+ Binary |= II->getRegisterInfo().getEncodingValue(ARM::LR);
+ else
+ // otherwise, set the return register
+ Binary |= getMachineOpValue(MI, 0);
+
+ emitWordLE(Binary);
+}
+
+unsigned ARMCodeEmitter::encodeVFPRd(const MachineInstr &MI,
+ unsigned OpIdx) const {
+ unsigned RegD = MI.getOperand(OpIdx).getReg();
+ unsigned Binary = 0;
+ bool isSPVFP = ARM::SPRRegClass.contains(RegD);
+ RegD = II->getRegisterInfo().getEncodingValue(RegD);
+ if (!isSPVFP)
+ Binary |= RegD << ARMII::RegRdShift;
+ else {
+ Binary |= ((RegD & 0x1E) >> 1) << ARMII::RegRdShift;
+ Binary |= (RegD & 0x01) << ARMII::D_BitShift;
+ }
+ return Binary;
+}
+
+unsigned ARMCodeEmitter::encodeVFPRn(const MachineInstr &MI,
+ unsigned OpIdx) const {
+ unsigned RegN = MI.getOperand(OpIdx).getReg();
+ unsigned Binary = 0;
+ bool isSPVFP = ARM::SPRRegClass.contains(RegN);
+ RegN = II->getRegisterInfo().getEncodingValue(RegN);
+ if (!isSPVFP)
+ Binary |= RegN << ARMII::RegRnShift;
+ else {
+ Binary |= ((RegN & 0x1E) >> 1) << ARMII::RegRnShift;
+ Binary |= (RegN & 0x01) << ARMII::N_BitShift;
+ }
+ return Binary;
+}
+
+unsigned ARMCodeEmitter::encodeVFPRm(const MachineInstr &MI,
+ unsigned OpIdx) const {
+ unsigned RegM = MI.getOperand(OpIdx).getReg();
+ unsigned Binary = 0;
+ bool isSPVFP = ARM::SPRRegClass.contains(RegM);
+ RegM = II->getRegisterInfo().getEncodingValue(RegM);
+ if (!isSPVFP)
+ Binary |= RegM;
+ else {
+ Binary |= ((RegM & 0x1E) >> 1);
+ Binary |= (RegM & 0x01) << ARMII::M_BitShift;
+ }
+ return Binary;
+}
+
+void ARMCodeEmitter::emitVFPArithInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ unsigned OpIdx = 0;
+ assert((Binary & ARMII::D_BitShift) == 0 &&
+ (Binary & ARMII::N_BitShift) == 0 &&
+ (Binary & ARMII::M_BitShift) == 0 && "VFP encoding bug!");
+
+ // Encode Dd / Sd.
+ Binary |= encodeVFPRd(MI, OpIdx++);
+
+ // If this is a two-address operand, skip it, e.g. FMACD.
+ if (MCID.getOperandConstraint(OpIdx, MCOI::TIED_TO) != -1)
+ ++OpIdx;
+
+ // Encode Dn / Sn.
+ if ((MCID.TSFlags & ARMII::FormMask) == ARMII::VFPBinaryFrm)
+ Binary |= encodeVFPRn(MI, OpIdx++);
+
+ if (OpIdx == MCID.getNumOperands() ||
+ MCID.OpInfo[OpIdx].isPredicate() ||
+ MCID.OpInfo[OpIdx].isOptionalDef()) {
+ // FCMPEZD etc. has only one operand.
+ emitWordLE(Binary);
+ return;
+ }
+
+ // Encode Dm / Sm.
+ Binary |= encodeVFPRm(MI, OpIdx);
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitVFPConversionInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ unsigned Form = MCID.TSFlags & ARMII::FormMask;
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ switch (Form) {
+ default: break;
+ case ARMII::VFPConv1Frm:
+ case ARMII::VFPConv2Frm:
+ case ARMII::VFPConv3Frm:
+ // Encode Dd / Sd.
+ Binary |= encodeVFPRd(MI, 0);
+ break;
+ case ARMII::VFPConv4Frm:
+ // Encode Dn / Sn.
+ Binary |= encodeVFPRn(MI, 0);
+ break;
+ case ARMII::VFPConv5Frm:
+ // Encode Dm / Sm.
+ Binary |= encodeVFPRm(MI, 0);
+ break;
+ }
+
+ switch (Form) {
+ default: break;
+ case ARMII::VFPConv1Frm:
+ // Encode Dm / Sm.
+ Binary |= encodeVFPRm(MI, 1);
+ break;
+ case ARMII::VFPConv2Frm:
+ case ARMII::VFPConv3Frm:
+ // Encode Dn / Sn.
+ Binary |= encodeVFPRn(MI, 1);
+ break;
+ case ARMII::VFPConv4Frm:
+ case ARMII::VFPConv5Frm:
+ // Encode Dd / Sd.
+ Binary |= encodeVFPRd(MI, 1);
+ break;
+ }
+
+ if (Form == ARMII::VFPConv5Frm)
+ // Encode Dn / Sn.
+ Binary |= encodeVFPRn(MI, 2);
+ else if (Form == ARMII::VFPConv3Frm)
+ // Encode Dm / Sm.
+ Binary |= encodeVFPRm(MI, 2);
+
+ emitWordLE(Binary);
+}
+
+void ARMCodeEmitter::emitVFPLoadStoreInstruction(const MachineInstr &MI) {
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ unsigned OpIdx = 0;
+
+ // Encode Dd / Sd.
+ Binary |= encodeVFPRd(MI, OpIdx++);
+
+ // Encode address base.
+ const MachineOperand &Base = MI.getOperand(OpIdx++);
+ Binary |= getMachineOpValue(MI, Base) << ARMII::RegRnShift;
+
+ // If there is a non-zero immediate offset, encode it.
+ if (Base.isReg()) {
+ const MachineOperand &Offset = MI.getOperand(OpIdx);
+ if (unsigned ImmOffs = ARM_AM::getAM5Offset(Offset.getImm())) {
+ if (ARM_AM::getAM5Op(Offset.getImm()) == ARM_AM::add)
+ Binary |= 1 << ARMII::U_BitShift;
+ Binary |= ImmOffs;
+ emitWordLE(Binary);
+ return;
+ }
+ }
+
+ // If immediate offset is omitted, default to +0.
+ Binary |= 1 << ARMII::U_BitShift;
+
+ emitWordLE(Binary);
+}
+
+void
+ARMCodeEmitter::emitVFPLoadStoreMultipleInstruction(const MachineInstr &MI) {
+ const MCInstrDesc &MCID = MI.getDesc();
+ bool IsUpdating = (MCID.TSFlags & ARMII::IndexModeMask) != 0;
+
+ // Part of binary is determined by TableGn.
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ // Set the conditional execution predicate
+ Binary |= II->getPredicate(&MI) << ARMII::CondShift;
+
+ // Skip operand 0 of an instruction with base register update.
+ unsigned OpIdx = 0;
+ if (IsUpdating)
+ ++OpIdx;
+
+ // Set base address operand
+ Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
+
+ // Set addressing mode by modifying bits U(23) and P(24)
+ ARM_AM::AMSubMode Mode = ARM_AM::getLoadStoreMultipleSubMode(MI.getOpcode());
+ Binary |= getAddrModeUPBits(ARM_AM::getAM4SubMode(Mode));
+
+ // Set bit W(21)
+ if (IsUpdating)
+ Binary |= 0x1 << ARMII::W_BitShift;
+
+ // First register is encoded in Dd.
+ Binary |= encodeVFPRd(MI, OpIdx+2);
+
+ // Count the number of registers.
+ unsigned NumRegs = 1;
+ for (unsigned i = OpIdx+3, e = MI.getNumOperands(); i != e; ++i) {
+ const MachineOperand &MO = MI.getOperand(i);
+ if (!MO.isReg() || MO.isImplicit())
+ break;
+ ++NumRegs;
+ }
+ // Bit 8 will be set if <list> is consecutive 64-bit registers (e.g., D0)
+ // Otherwise, it will be 0, in the case of 32-bit registers.
+ if(Binary & 0x100)
+ Binary |= NumRegs * 2;
+ else
+ Binary |= NumRegs;
+
+ emitWordLE(Binary);
+}
+
+unsigned ARMCodeEmitter::encodeNEONRd(const MachineInstr &MI,
+ unsigned OpIdx) const {
+ unsigned RegD = MI.getOperand(OpIdx).getReg();
+ unsigned Binary = 0;
+ RegD = II->getRegisterInfo().getEncodingValue(RegD);
+ Binary |= (RegD & 0xf) << ARMII::RegRdShift;
+ Binary |= ((RegD >> 4) & 1) << ARMII::D_BitShift;
+ return Binary;
+}
+
+unsigned ARMCodeEmitter::encodeNEONRn(const MachineInstr &MI,
+ unsigned OpIdx) const {
+ unsigned RegN = MI.getOperand(OpIdx).getReg();
+ unsigned Binary = 0;
+ RegN = II->getRegisterInfo().getEncodingValue(RegN);
+ Binary |= (RegN & 0xf) << ARMII::RegRnShift;
+ Binary |= ((RegN >> 4) & 1) << ARMII::N_BitShift;
+ return Binary;
+}
+
+unsigned ARMCodeEmitter::encodeNEONRm(const MachineInstr &MI,
+ unsigned OpIdx) const {
+ unsigned RegM = MI.getOperand(OpIdx).getReg();
+ unsigned Binary = 0;
+ RegM = II->getRegisterInfo().getEncodingValue(RegM);
+ Binary |= (RegM & 0xf);
+ Binary |= ((RegM >> 4) & 1) << ARMII::M_BitShift;
+ return Binary;
+}
+
+/// convertNEONDataProcToThumb - Convert the ARM mode encoding for a NEON
+/// data-processing instruction to the corresponding Thumb encoding.
+static unsigned convertNEONDataProcToThumb(unsigned Binary) {
+ assert((Binary & 0xfe000000) == 0xf2000000 &&
+ "not an ARM NEON data-processing instruction");
+ unsigned UBit = (Binary >> 24) & 1;
+ return 0xef000000 | (UBit << 28) | (Binary & 0xffffff);
+}
+
+void ARMCodeEmitter::emitNEONLaneInstruction(const MachineInstr &MI) {
+ unsigned Binary = getBinaryCodeForInstr(MI);
+
+ unsigned RegTOpIdx, RegNOpIdx, LnOpIdx;
+ const MCInstrDesc &MCID = MI.getDesc();
+ if ((MCID.TSFlags & ARMII::FormMask) == ARMII::NGetLnFrm) {
+ RegTOpIdx = 0;
+ RegNOpIdx = 1;
+ LnOpIdx = 2;
+ } else { // ARMII::NSetLnFrm
+ RegTOpIdx = 2;
+ RegNOpIdx = 0;
+ LnOpIdx = 3;
+ }
+
+ // Set the conditional execution predicate
+ Binary |= (IsThumb ? ARMCC::AL : II->getPredicate(&MI)) << ARMII::CondShift;
+
+ unsigned RegT = MI.getOperand(RegTOpIdx).getReg();
+ RegT = II->getRegisterInfo().getEncodingValue(RegT);
+ Binary |= (RegT << ARMII::RegRdShift);
+ Binary |= encodeNEONRn(MI, RegNOpIdx);
+
+ unsigned LaneShift;
+ if ((Binary & (1 << 22)) != 0)
+ LaneShift = 0; // 8-bit elements
+ else if ((Binary & (1 << 5)) != 0)
+ LaneShift = 1; // 16-bit elements
+ else
+ LaneShift = 2; // 32-bit elements
+
+ unsigned Lane = MI.getOperand(LnOpIdx).getImm() << LaneShift;
+ unsigned Opc1 = Lane >> 2;
+ unsigned Opc2 = Lane & 3;
+ assert((Opc1 & 3) == 0 && "out-of-range lane number operand");
projects / oota-llvm.git / commitdiff