//
//===----------------------------------------------------------------------===//
//
-// This pass lowers instrprof_increment intrinsics emitted by a frontend for
-// profiling. It also builds the data structures and initialization code needed
-// for updating execution counts and emitting the profile at runtime.
+// This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
+// It also builds the data structures and initialization code needed for
+// updating execution counts and emitting the profile at runtime.
//
//===----------------------------------------------------------------------===//
-#include "llvm/ProfileData/InstrProf.h"
-#include "llvm/Transforms/Instrumentation.h"
-
#include "llvm/ADT/Triple.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
using namespace llvm;
private:
InstrProfOptions Options;
Module *M;
- DenseMap<GlobalVariable *, GlobalVariable *> RegionCounters;
+ typedef struct PerFunctionProfileData {
+ uint32_t NumValueSites[IPVK_Last+1];
+ GlobalVariable* RegionCounters;
+ GlobalVariable* DataVar;
+ PerFunctionProfileData() : RegionCounters(nullptr), DataVar(nullptr) {
+ memset(NumValueSites, 0, sizeof(uint32_t) * (IPVK_Last+1));
+ }
+ } PerFunctionProfileData;
+ DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
std::vector<Value *> UsedVars;
bool isMachO() const {
return getInstrProfCoverageSectionName(isMachO());
}
+ /// Count the number of instrumented value sites for the function.
+ void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
+
+ /// Replace instrprof_value_profile with a call to runtime library.
+ void lowerValueProfileInst(InstrProfValueProfileInst *Ins);
+
/// Replace instrprof_increment with an increment of the appropriate value.
void lowerIncrement(InstrProfIncrementInst *Inc);
bool MadeChange = false;
this->M = &M;
- RegionCounters.clear();
+ ProfileDataMap.clear();
UsedVars.clear();
+ // We did not know how many value sites there would be inside
+ // the instrumented function. This is counting the number of instrumented
+ // target value sites to enter it as field in the profile data variable.
for (Function &F : M)
for (BasicBlock &BB : F)
for (auto I = BB.begin(), E = BB.end(); I != E;)
- if (auto *Inc = dyn_cast<InstrProfIncrementInst>(I++)) {
+ if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I++))
+ computeNumValueSiteCounts(Ind);
+
+ for (Function &F : M)
+ for (BasicBlock &BB : F)
+ for (auto I = BB.begin(), E = BB.end(); I != E;) {
+ auto Instr = I++;
+ if (auto *Inc = dyn_cast<InstrProfIncrementInst>(Instr)) {
lowerIncrement(Inc);
MadeChange = true;
+ } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
+ lowerValueProfileInst(Ind);
+ MadeChange = true;
}
+ }
+
if (GlobalVariable *Coverage =
M.getNamedGlobal(getCoverageMappingVarName())) {
lowerCoverageData(Coverage);
MadeChange = true;
}
+
if (!MadeChange)
return false;
return true;
}
+static Constant *getOrInsertValueProfilingCall(Module &M) {
+ LLVMContext &Ctx = M.getContext();
+ auto *ReturnTy = Type::getVoidTy(M.getContext());
+ Type *ParamTypes[] = {
+#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
+#include "llvm/ProfileData/InstrProfData.inc"
+ };
+ auto *ValueProfilingCallTy =
+ FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
+ return M.getOrInsertFunction(getInstrProfValueProfFuncName(),
+ ValueProfilingCallTy);
+}
+
+void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
+
+ GlobalVariable *Name = Ind->getName();
+ uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
+ uint64_t Index = Ind->getIndex()->getZExtValue();
+ auto It = ProfileDataMap.find(Name);
+ if (It == ProfileDataMap.end()) {
+ PerFunctionProfileData PD;
+ PD.NumValueSites[ValueKind] = Index + 1;
+ ProfileDataMap[Name] = PD;
+ } else if (It->second.NumValueSites[ValueKind] <= Index)
+ It->second.NumValueSites[ValueKind] = Index + 1;
+}
+
+void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
+
+ GlobalVariable *Name = Ind->getName();
+ auto It = ProfileDataMap.find(Name);
+ assert(It != ProfileDataMap.end() && It->second.DataVar &&
+ "value profiling detected in function with no counter incerement");
+
+ GlobalVariable *DataVar = It->second.DataVar;
+ uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
+ uint64_t Index = Ind->getIndex()->getZExtValue();
+ for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
+ Index += It->second.NumValueSites[Kind];
+
+ IRBuilder<> Builder(Ind);
+ Value* Args[3] = {Ind->getTargetValue(),
+ Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
+ Builder.getInt32(Index)};
+ Ind->replaceAllUsesWith(
+ Builder.CreateCall(getOrInsertValueProfilingCall(*M), Args));
+ Ind->eraseFromParent();
+}
+
void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
GlobalVariable *Name = cast<GlobalVariable>(V);
// If we have region counters for this name, we've already handled it.
- auto It = RegionCounters.find(Name);
- if (It != RegionCounters.end())
- continue;
+ auto It = ProfileDataMap.find(Name);
+ if (It != ProfileDataMap.end())
+ if (It->second.RegionCounters)
+ continue;
// Move the name variable to the right section.
Name->setSection(getNameSection());
return (Prefix + Name).str();
}
+static inline bool shouldRecordFunctionAddr(Function *F) {
+ // Check the linkage
+ if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
+ !F->hasAvailableExternallyLinkage())
+ return true;
+ // Check uses of this function for other than direct calls or invokes to it.
+ return F->hasAddressTaken();
+}
+
GlobalVariable *
InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
- GlobalVariable *Name = Inc->getName();
- auto It = RegionCounters.find(Name);
- if (It != RegionCounters.end())
- return It->second;
+ GlobalVariable *NamePtr = Inc->getName();
+ auto It = ProfileDataMap.find(NamePtr);
+ PerFunctionProfileData PD;
+ if (It != ProfileDataMap.end()) {
+ if (It->second.RegionCounters)
+ return It->second.RegionCounters;
+ PD = It->second;
+ }
// Move the name variable to the right section. Place them in a COMDAT group
// if the associated function is a COMDAT. This will make sure that
if (Fn->hasComdat())
ProfileVarsComdat = M->getOrInsertComdat(
StringRef(getVarName(Inc, getInstrProfComdatPrefix())));
- Name->setSection(getNameSection());
- Name->setAlignment(1);
- Name->setComdat(ProfileVarsComdat);
+ NamePtr->setSection(getNameSection());
+ NamePtr->setAlignment(1);
+ NamePtr->setComdat(ProfileVarsComdat);
uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
LLVMContext &Ctx = M->getContext();
ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
// Create the counters variable.
- auto *Counters =
- new GlobalVariable(*M, CounterTy, false, Name->getLinkage(),
+ auto *CounterPtr =
+ new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
Constant::getNullValue(CounterTy),
getVarName(Inc, getInstrProfCountersVarPrefix()));
- Counters->setVisibility(Name->getVisibility());
- Counters->setSection(getCountersSection());
- Counters->setAlignment(8);
- Counters->setComdat(ProfileVarsComdat);
-
- RegionCounters[Inc->getName()] = Counters;
+ CounterPtr->setVisibility(NamePtr->getVisibility());
+ CounterPtr->setSection(getCountersSection());
+ CounterPtr->setAlignment(8);
+ CounterPtr->setComdat(ProfileVarsComdat);
// Create data variable.
- auto *NameArrayTy = Name->getType()->getPointerElementType();
- auto *Int32Ty = Type::getInt32Ty(Ctx);
- auto *Int64Ty = Type::getInt64Ty(Ctx);
auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
- auto *Int64PtrTy = Type::getInt64PtrTy(Ctx);
-
- Type *DataTypes[] = {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int64PtrTy};
+ auto *Int16Ty = Type::getInt16Ty(Ctx);
+ auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last+1);
+ Type *DataTypes[] = {
+ #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
+ #include "llvm/ProfileData/InstrProfData.inc"
+ };
auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
+
+ Constant *FunctionAddr = shouldRecordFunctionAddr(Fn) ?
+ ConstantExpr::getBitCast(Fn, Int8PtrTy) :
+ ConstantPointerNull::get(Int8PtrTy);
+
+ Constant *Int16ArrayVals[IPVK_Last+1];
+ for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
+ Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
+
Constant *DataVals[] = {
- ConstantInt::get(Int32Ty, NameArrayTy->getArrayNumElements()),
- ConstantInt::get(Int32Ty, NumCounters),
- ConstantInt::get(Int64Ty, Inc->getHash()->getZExtValue()),
- ConstantExpr::getBitCast(Name, Int8PtrTy),
- ConstantExpr::getBitCast(Counters, Int64PtrTy)};
- auto *Data = new GlobalVariable(*M, DataTy, true, Name->getLinkage(),
+ #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
+ #include "llvm/ProfileData/InstrProfData.inc"
+ };
+ auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
ConstantStruct::get(DataTy, DataVals),
getVarName(Inc, getInstrProfDataVarPrefix()));
- Data->setVisibility(Name->getVisibility());
+ Data->setVisibility(NamePtr->getVisibility());
Data->setSection(getDataSection());
Data->setAlignment(8);
Data->setComdat(ProfileVarsComdat);
+ PD.RegionCounters = CounterPtr;
+ PD.DataVar = Data;
+ ProfileDataMap[NamePtr] = PD;
+
// Mark the data variable as used so that it isn't stripped out.
UsedVars.push_back(Data);
- return Counters;
+ return CounterPtr;
}
void InstrProfiling::emitRegistration() {
LLVMUsed =
new GlobalVariable(*M, ATy, false, GlobalValue::AppendingLinkage,
ConstantArray::get(ATy, MergedVars), "llvm.used");
-
LLVMUsed->setSection("llvm.metadata");
}
projects / oota-llvm.git / blobdiff