//
//===----------------------------------------------------------------------===//
//
-// This pass lowers instrprof_increment intrinsics emitted by a frontend for
-// profiling. It also builds the data structures and initialization code needed
-// for updating execution counts and emitting the profile at runtime.
+// This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
+// It also builds the data structures and initialization code needed for
+// updating execution counts and emitting the profile at runtime.
//
//===----------------------------------------------------------------------===//
-#include "llvm/Transforms/Instrumentation.h"
-
#include "llvm/ADT/Triple.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
using namespace llvm;
private:
InstrProfOptions Options;
Module *M;
- DenseMap<GlobalVariable *, GlobalVariable *> RegionCounters;
+ typedef struct PerFunctionProfileData {
+ uint32_t NumValueSites[IPVK_Last+1];
+ GlobalVariable* RegionCounters;
+ GlobalVariable* DataVar;
+ PerFunctionProfileData() : RegionCounters(nullptr), DataVar(nullptr) {
+ memset(NumValueSites, 0, sizeof(uint32_t) * (IPVK_Last+1));
+ }
+ } PerFunctionProfileData;
+ DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
std::vector<Value *> UsedVars;
bool isMachO() const {
/// Get the section name for the counter variables.
StringRef getCountersSection() const {
- return isMachO() ? "__DATA,__llvm_prf_cnts" : "__llvm_prf_cnts";
+ return getInstrProfCountersSectionName(isMachO());
}
/// Get the section name for the name variables.
StringRef getNameSection() const {
- return isMachO() ? "__DATA,__llvm_prf_names" : "__llvm_prf_names";
+ return getInstrProfNameSectionName(isMachO());
}
/// Get the section name for the profile data variables.
StringRef getDataSection() const {
- return isMachO() ? "__DATA,__llvm_prf_data" : "__llvm_prf_data";
+ return getInstrProfDataSectionName(isMachO());
}
/// Get the section name for the coverage mapping data.
StringRef getCoverageSection() const {
- return isMachO() ? "__DATA,__llvm_covmap" : "__llvm_covmap";
+ return getInstrProfCoverageSectionName(isMachO());
}
+ /// Count the number of instrumented value sites for the function.
+ void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
+
+ /// Replace instrprof_value_profile with a call to runtime library.
+ void lowerValueProfileInst(InstrProfValueProfileInst *Ins);
+
/// Replace instrprof_increment with an increment of the appropriate value.
void lowerIncrement(InstrProfIncrementInst *Inc);
bool MadeChange = false;
this->M = &M;
- RegionCounters.clear();
+ ProfileDataMap.clear();
UsedVars.clear();
+ // We did not know how many value sites there would be inside
+ // the instrumented function. This is counting the number of instrumented
+ // target value sites to enter it as field in the profile data variable.
for (Function &F : M)
for (BasicBlock &BB : F)
for (auto I = BB.begin(), E = BB.end(); I != E;)
- if (auto *Inc = dyn_cast<InstrProfIncrementInst>(I++)) {
+ if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I++))
+ computeNumValueSiteCounts(Ind);
+
+ for (Function &F : M)
+ for (BasicBlock &BB : F)
+ for (auto I = BB.begin(), E = BB.end(); I != E;) {
+ auto Instr = I++;
+ if (auto *Inc = dyn_cast<InstrProfIncrementInst>(Instr)) {
lowerIncrement(Inc);
MadeChange = true;
+ } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
+ lowerValueProfileInst(Ind);
+ MadeChange = true;
}
- if (GlobalVariable *Coverage = M.getNamedGlobal("__llvm_coverage_mapping")) {
+ }
+
+ if (GlobalVariable *Coverage =
+ M.getNamedGlobal(getCoverageMappingVarName())) {
lowerCoverageData(Coverage);
MadeChange = true;
}
+
if (!MadeChange)
return false;
return true;
}
+static Constant *getOrInsertValueProfilingCall(Module &M) {
+ auto *VoidTy = Type::getVoidTy(M.getContext());
+ auto *VoidPtrTy = Type::getInt8PtrTy(M.getContext());
+ auto *Int32Ty = Type::getInt32Ty(M.getContext());
+ auto *Int64Ty = Type::getInt64Ty(M.getContext());
+ Type *ArgTypes[] = {Int64Ty, VoidPtrTy, Int32Ty};
+ auto *ValueProfilingCallTy =
+ FunctionType::get(VoidTy, makeArrayRef(ArgTypes), false);
+ return M.getOrInsertFunction("__llvm_profile_instrument_target",
+ ValueProfilingCallTy);
+}
+
+void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
+
+ GlobalVariable *Name = Ind->getName();
+ uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
+ uint64_t Index = Ind->getIndex()->getZExtValue();
+ auto It = ProfileDataMap.find(Name);
+ if (It == ProfileDataMap.end()) {
+ PerFunctionProfileData PD;
+ PD.NumValueSites[ValueKind] = Index + 1;
+ ProfileDataMap[Name] = PD;
+ } else if (It->second.NumValueSites[ValueKind] <= Index)
+ It->second.NumValueSites[ValueKind] = Index + 1;
+}
+
+void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
+
+ GlobalVariable *Name = Ind->getName();
+ auto It = ProfileDataMap.find(Name);
+ assert(It != ProfileDataMap.end() && It->second.DataVar &&
+ "value profiling detected in function with no counter incerement");
+
+ GlobalVariable *DataVar = It->second.DataVar;
+ uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
+ uint64_t Index = Ind->getIndex()->getZExtValue();
+ for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
+ Index += It->second.NumValueSites[Kind];
+
+ IRBuilder<> Builder(Ind);
+ Value* Args[3] = {Ind->getTargetValue(),
+ Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
+ Builder.getInt32(Index)};
+ Ind->replaceAllUsesWith(
+ Builder.CreateCall(getOrInsertValueProfilingCall(*M), Args));
+ Ind->eraseFromParent();
+}
+
void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
- IRBuilder<> Builder(Inc->getParent(), *Inc);
+ IRBuilder<> Builder(Inc);
uint64_t Index = Inc->getIndex()->getZExtValue();
Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
Value *Count = Builder.CreateLoad(Addr, "pgocount");
GlobalVariable *Name = cast<GlobalVariable>(V);
// If we have region counters for this name, we've already handled it.
- auto It = RegionCounters.find(Name);
- if (It != RegionCounters.end())
- continue;
+ auto It = ProfileDataMap.find(Name);
+ if (It != ProfileDataMap.end())
+ if (It->second.RegionCounters)
+ continue;
// Move the name variable to the right section.
Name->setSection(getNameSection());
}
/// Get the name of a profiling variable for a particular function.
-static std::string getVarName(InstrProfIncrementInst *Inc, StringRef VarName) {
+static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
auto *Arr = cast<ConstantDataArray>(Inc->getName()->getInitializer());
StringRef Name = Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
- return ("__llvm_profile_" + VarName + "_" + Name).str();
+ return (Prefix + Name).str();
+}
+
+static inline bool shouldRecordFunctionAddr(Function *F) {
+ // Check the linkage
+ if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
+ !F->hasAvailableExternallyLinkage())
+ return true;
+ // Check uses of this function for other than direct calls or invokes to it.
+ return F->hasAddressTaken();
}
GlobalVariable *
InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
- GlobalVariable *Name = Inc->getName();
- auto It = RegionCounters.find(Name);
- if (It != RegionCounters.end())
- return It->second;
+ GlobalVariable *NamePtr = Inc->getName();
+ auto It = ProfileDataMap.find(NamePtr);
+ PerFunctionProfileData PD;
+ if (It != ProfileDataMap.end()) {
+ if (It->second.RegionCounters)
+ return It->second.RegionCounters;
+ PD = It->second;
+ }
// Move the name variable to the right section. Place them in a COMDAT group
// if the associated function is a COMDAT. This will make sure that
Function *Fn = Inc->getParent()->getParent();
Comdat *ProfileVarsComdat = nullptr;
if (Fn->hasComdat())
- ProfileVarsComdat = M->getOrInsertComdat(StringRef(getVarName(Inc, "vars")));
- Name->setSection(getNameSection());
- Name->setAlignment(1);
- Name->setComdat(ProfileVarsComdat);
+ ProfileVarsComdat = M->getOrInsertComdat(
+ StringRef(getVarName(Inc, getInstrProfComdatPrefix())));
+ NamePtr->setSection(getNameSection());
+ NamePtr->setAlignment(1);
+ NamePtr->setComdat(ProfileVarsComdat);
uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
LLVMContext &Ctx = M->getContext();
ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
// Create the counters variable.
- auto *Counters = new GlobalVariable(*M, CounterTy, false, Name->getLinkage(),
- Constant::getNullValue(CounterTy),
- getVarName(Inc, "counters"));
- Counters->setVisibility(Name->getVisibility());
- Counters->setSection(getCountersSection());
- Counters->setAlignment(8);
- Counters->setComdat(ProfileVarsComdat);
-
- RegionCounters[Inc->getName()] = Counters;
+ auto *CounterPtr =
+ new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
+ Constant::getNullValue(CounterTy),
+ getVarName(Inc, getInstrProfCountersVarPrefix()));
+ CounterPtr->setVisibility(NamePtr->getVisibility());
+ CounterPtr->setSection(getCountersSection());
+ CounterPtr->setAlignment(8);
+ CounterPtr->setComdat(ProfileVarsComdat);
// Create data variable.
- auto *NameArrayTy = Name->getType()->getPointerElementType();
- auto *Int32Ty = Type::getInt32Ty(Ctx);
- auto *Int64Ty = Type::getInt64Ty(Ctx);
auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
- auto *Int64PtrTy = Type::getInt64PtrTy(Ctx);
-
- Type *DataTypes[] = {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int64PtrTy};
+ auto *Int16Ty = Type::getInt16Ty(Ctx);
+ auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last+1);
+ Type *DataTypes[] = {
+ #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
+ #include "llvm/ProfileData/InstrProfData.inc"
+ };
auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
+
+ Constant *FunctionAddr = shouldRecordFunctionAddr(Fn) ?
+ ConstantExpr::getBitCast(Fn, Int8PtrTy) :
+ ConstantPointerNull::get(Int8PtrTy);
+
+ Constant *Int16ArrayVals[IPVK_Last+1];
+ for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
+ Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
+
Constant *DataVals[] = {
- ConstantInt::get(Int32Ty, NameArrayTy->getArrayNumElements()),
- ConstantInt::get(Int32Ty, NumCounters),
- ConstantInt::get(Int64Ty, Inc->getHash()->getZExtValue()),
- ConstantExpr::getBitCast(Name, Int8PtrTy),
- ConstantExpr::getBitCast(Counters, Int64PtrTy)};
- auto *Data = new GlobalVariable(*M, DataTy, true, Name->getLinkage(),
+ #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
+ #include "llvm/ProfileData/InstrProfData.inc"
+ };
+ auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
ConstantStruct::get(DataTy, DataVals),
- getVarName(Inc, "data"));
- Data->setVisibility(Name->getVisibility());
+ getVarName(Inc, getInstrProfDataVarPrefix()));
+ Data->setVisibility(NamePtr->getVisibility());
Data->setSection(getDataSection());
Data->setAlignment(8);
Data->setComdat(ProfileVarsComdat);
+ PD.RegionCounters = CounterPtr;
+ PD.DataVar = Data;
+ ProfileDataMap[NamePtr] = PD;
+
// Mark the data variable as used so that it isn't stripped out.
UsedVars.push_back(Data);
- return Counters;
+ return CounterPtr;
}
void InstrProfiling::emitRegistration() {
if (Triple(M->getTargetTriple()).isOSDarwin())
return;
+ // Use linker script magic to get data/cnts/name start/end.
+ if (Triple(M->getTargetTriple()).isOSLinux() ||
+ Triple(M->getTargetTriple()).isOSFreeBSD())
+ return;
+
// Construct the function.
auto *VoidTy = Type::getVoidTy(M->getContext());
auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
auto *RegisterFTy = FunctionType::get(VoidTy, false);
auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
- "__llvm_profile_register_functions", M);
+ getInstrProfRegFuncsName(), M);
RegisterF->setUnnamedAddr(true);
- if (Options.NoRedZone)
- RegisterF->addFnAttr(Attribute::NoRedZone);
+ if (Options.NoRedZone) RegisterF->addFnAttr(Attribute::NoRedZone);
auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
auto *RuntimeRegisterF =
Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
- "__llvm_profile_register_function", M);
+ getInstrProfRegFuncName(), M);
IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
for (Value *Data : UsedVars)
}
void InstrProfiling::emitRuntimeHook() {
- const char *const RuntimeVarName = "__llvm_profile_runtime";
- const char *const RuntimeUserName = "__llvm_profile_runtime_user";
- // If the module's provided its own runtime, we don't need to do anything.
- if (M->getGlobalVariable(RuntimeVarName))
+ // We expect the linker to be invoked with -u<hook_var> flag for linux,
+ // for which case there is no need to emit the user function.
+ if (Triple(M->getTargetTriple()).isOSLinux())
return;
+ // If the module's provided its own runtime, we don't need to do anything.
+ if (M->getGlobalVariable(getInstrProfRuntimeHookVarName())) return;
+
// Declare an external variable that will pull in the runtime initialization.
auto *Int32Ty = Type::getInt32Ty(M->getContext());
auto *Var =
new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
- nullptr, RuntimeVarName);
+ nullptr, getInstrProfRuntimeHookVarName());
// Make a function that uses it.
- auto *User =
- Function::Create(FunctionType::get(Int32Ty, false),
- GlobalValue::LinkOnceODRLinkage, RuntimeUserName, M);
+ auto *User = Function::Create(FunctionType::get(Int32Ty, false),
+ GlobalValue::LinkOnceODRLinkage,
+ getInstrProfRuntimeHookVarUseFuncName(), M);
User->addFnAttr(Attribute::NoInline);
- if (Options.NoRedZone)
- User->addFnAttr(Attribute::NoRedZone);
+ if (Options.NoRedZone) User->addFnAttr(Attribute::NoRedZone);
User->setVisibility(GlobalValue::HiddenVisibility);
IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
LLVMUsed =
new GlobalVariable(*M, ATy, false, GlobalValue::AppendingLinkage,
ConstantArray::get(ATy, MergedVars), "llvm.used");
-
LLVMUsed->setSection("llvm.metadata");
}
void InstrProfiling::emitInitialization() {
std::string InstrProfileOutput = Options.InstrProfileOutput;
- Constant *RegisterF = M->getFunction("__llvm_profile_register_functions");
- if (!RegisterF && InstrProfileOutput.empty())
- return;
+ Constant *RegisterF = M->getFunction(getInstrProfRegFuncsName());
+ if (!RegisterF && InstrProfileOutput.empty()) return;
// Create the initialization function.
auto *VoidTy = Type::getVoidTy(M->getContext());
- auto *F =
- Function::Create(FunctionType::get(VoidTy, false),
- GlobalValue::InternalLinkage, "__llvm_profile_init", M);
+ auto *F = Function::Create(FunctionType::get(VoidTy, false),
+ GlobalValue::InternalLinkage,
+ getInstrProfInitFuncName(), M);
F->setUnnamedAddr(true);
F->addFnAttr(Attribute::NoInline);
- if (Options.NoRedZone)
- F->addFnAttr(Attribute::NoRedZone);
+ if (Options.NoRedZone) F->addFnAttr(Attribute::NoRedZone);
// Add the basic block and the necessary calls.
IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
if (!InstrProfileOutput.empty()) {
auto *Int8PtrTy = Type::getInt8PtrTy(M->getContext());
auto *SetNameTy = FunctionType::get(VoidTy, Int8PtrTy, false);
- auto *SetNameF =
- Function::Create(SetNameTy, GlobalValue::ExternalLinkage,
- "__llvm_profile_override_default_filename", M);
+ auto *SetNameF = Function::Create(SetNameTy, GlobalValue::ExternalLinkage,
+ getInstrProfFileOverriderFuncName(), M);
// Create variable for profile name.
Constant *ProfileNameConst =
projects / oota-llvm.git / blobdiff