1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass lowers instrprof_increment intrinsics emitted by a frontend for
11 // profiling. It also builds the data structures and initialization code needed
12 // for updating execution counts and emitting the profile at runtime.
14 //===----------------------------------------------------------------------===//
16 #include "llvm/ProfileData/InstrProf.h"
17 #include "llvm/Transforms/Instrumentation.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/IR/IRBuilder.h"
21 #include "llvm/IR/IntrinsicInst.h"
22 #include "llvm/IR/Module.h"
23 #include "llvm/Transforms/Utils/ModuleUtils.h"
27 #define DEBUG_TYPE "instrprof"
31 class InstrProfiling : public ModulePass {
35 InstrProfiling() : ModulePass(ID) {}
37 InstrProfiling(const InstrProfOptions &Options)
38 : ModulePass(ID), Options(Options) {}
40 const char *getPassName() const override {
41 return "Frontend instrumentation-based coverage lowering";
44 bool runOnModule(Module &M) override;
46 void getAnalysisUsage(AnalysisUsage &AU) const override {
51 InstrProfOptions Options;
53 DenseMap<GlobalVariable *, GlobalVariable *> RegionCounters;
54 std::vector<Value *> UsedVars;
56 bool isMachO() const {
57 return Triple(M->getTargetTriple()).isOSBinFormatMachO();
60 /// Get the section name for the counter variables.
61 StringRef getCountersSection() const {
62 return getInstrProfCountersSectionName(isMachO());
65 /// Get the section name for the name variables.
66 StringRef getNameSection() const {
67 return getInstrProfNameSectionName(isMachO());
70 /// Get the section name for the profile data variables.
71 StringRef getDataSection() const {
72 return getInstrProfDataSectionName(isMachO());
75 /// Get the section name for the coverage mapping data.
76 StringRef getCoverageSection() const {
77 return getInstrProfCoverageSectionName(isMachO());
80 /// Replace instrprof_increment with an increment of the appropriate value.
81 void lowerIncrement(InstrProfIncrementInst *Inc);
83 /// Set up the section and uses for coverage data and its references.
84 void lowerCoverageData(GlobalVariable *CoverageData);
86 /// Get the region counters for an increment, creating them if necessary.
88 /// If the counter array doesn't yet exist, the profile data variables
89 /// referring to them will also be created.
90 GlobalVariable *getOrCreateRegionCounters(InstrProfIncrementInst *Inc);
92 /// Emit runtime registration functions for each profile data variable.
93 void emitRegistration();
95 /// Emit the necessary plumbing to pull in the runtime initialization.
96 void emitRuntimeHook();
98 /// Add uses of our data variables and runtime hook.
101 /// Create a static initializer for our data, on platforms that need it,
102 /// and for any profile output file that was specified.
103 void emitInitialization();
106 } // anonymous namespace
108 char InstrProfiling::ID = 0;
109 INITIALIZE_PASS(InstrProfiling, "instrprof",
110 "Frontend instrumentation-based coverage lowering.", false,
113 ModulePass *llvm::createInstrProfilingPass(const InstrProfOptions &Options) {
114 return new InstrProfiling(Options);
117 bool InstrProfiling::runOnModule(Module &M) {
118 bool MadeChange = false;
121 RegionCounters.clear();
124 for (Function &F : M)
125 for (BasicBlock &BB : F)
126 for (auto I = BB.begin(), E = BB.end(); I != E;)
127 if (auto *Inc = dyn_cast<InstrProfIncrementInst>(I++)) {
131 if (GlobalVariable *Coverage =
132 M.getNamedGlobal(getCoverageMappingVarName())) {
133 lowerCoverageData(Coverage);
142 emitInitialization();
146 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
147 GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
149 IRBuilder<> Builder(Inc);
150 uint64_t Index = Inc->getIndex()->getZExtValue();
151 Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
152 Value *Count = Builder.CreateLoad(Addr, "pgocount");
153 Count = Builder.CreateAdd(Count, Builder.getInt64(1));
154 Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
155 Inc->eraseFromParent();
158 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageData) {
159 CoverageData->setSection(getCoverageSection());
160 CoverageData->setAlignment(8);
162 Constant *Init = CoverageData->getInitializer();
163 // We're expecting { i32, i32, i32, i32, [n x { i8*, i32, i32 }], [m x i8] }
164 // for some C. If not, the frontend's given us something broken.
165 assert(Init->getNumOperands() == 6 && "bad number of fields in coverage map");
166 assert(isa<ConstantArray>(Init->getAggregateElement(4)) &&
167 "invalid function list in coverage map");
168 ConstantArray *Records = cast<ConstantArray>(Init->getAggregateElement(4));
169 for (unsigned I = 0, E = Records->getNumOperands(); I < E; ++I) {
170 Constant *Record = Records->getOperand(I);
171 Value *V = const_cast<Value *>(Record->getOperand(0))->stripPointerCasts();
173 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
174 GlobalVariable *Name = cast<GlobalVariable>(V);
176 // If we have region counters for this name, we've already handled it.
177 auto It = RegionCounters.find(Name);
178 if (It != RegionCounters.end())
181 // Move the name variable to the right section.
182 Name->setSection(getNameSection());
183 Name->setAlignment(1);
187 /// Get the name of a profiling variable for a particular function.
188 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
189 auto *Arr = cast<ConstantDataArray>(Inc->getName()->getInitializer());
190 StringRef Name = Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
191 return (Prefix + Name).str();
195 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
196 GlobalVariable *Name = Inc->getName();
197 auto It = RegionCounters.find(Name);
198 if (It != RegionCounters.end())
201 // Move the name variable to the right section. Place them in a COMDAT group
202 // if the associated function is a COMDAT. This will make sure that
203 // only one copy of counters of the COMDAT function will be emitted after
205 Function *Fn = Inc->getParent()->getParent();
206 Comdat *ProfileVarsComdat = nullptr;
208 ProfileVarsComdat = M->getOrInsertComdat(
209 StringRef(getVarName(Inc, getInstrProfComdatPrefix())));
210 Name->setSection(getNameSection());
211 Name->setAlignment(1);
212 Name->setComdat(ProfileVarsComdat);
214 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
215 LLVMContext &Ctx = M->getContext();
216 ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
218 // Create the counters variable.
220 new GlobalVariable(*M, CounterTy, false, Name->getLinkage(),
221 Constant::getNullValue(CounterTy),
222 getVarName(Inc, getInstrProfCountersVarPrefix()));
223 Counters->setVisibility(Name->getVisibility());
224 Counters->setSection(getCountersSection());
225 Counters->setAlignment(8);
226 Counters->setComdat(ProfileVarsComdat);
228 RegionCounters[Inc->getName()] = Counters;
230 // Create data variable.
231 auto *NameArrayTy = Name->getType()->getPointerElementType();
232 auto *Int32Ty = Type::getInt32Ty(Ctx);
233 auto *Int64Ty = Type::getInt64Ty(Ctx);
234 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
235 auto *Int64PtrTy = Type::getInt64PtrTy(Ctx);
237 Type *DataTypes[] = {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int64PtrTy};
238 auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
239 Constant *DataVals[] = {
240 ConstantInt::get(Int32Ty, NameArrayTy->getArrayNumElements()),
241 ConstantInt::get(Int32Ty, NumCounters),
242 ConstantInt::get(Int64Ty, Inc->getHash()->getZExtValue()),
243 ConstantExpr::getBitCast(Name, Int8PtrTy),
244 ConstantExpr::getBitCast(Counters, Int64PtrTy)};
245 auto *Data = new GlobalVariable(*M, DataTy, true, Name->getLinkage(),
246 ConstantStruct::get(DataTy, DataVals),
247 getVarName(Inc, getInstrProfDataVarPrefix()));
248 Data->setVisibility(Name->getVisibility());
249 Data->setSection(getDataSection());
250 Data->setAlignment(8);
251 Data->setComdat(ProfileVarsComdat);
253 // Mark the data variable as used so that it isn't stripped out.
254 UsedVars.push_back(Data);
259 void InstrProfiling::emitRegistration() {
260 // Don't do this for Darwin. compiler-rt uses linker magic.
261 if (Triple(M->getTargetTriple()).isOSDarwin())
264 // Use linker script magic to get data/cnts/name start/end.
265 if (Triple(M->getTargetTriple()).isOSLinux() ||
266 Triple(M->getTargetTriple()).isOSFreeBSD())
269 // Construct the function.
270 auto *VoidTy = Type::getVoidTy(M->getContext());
271 auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
272 auto *RegisterFTy = FunctionType::get(VoidTy, false);
273 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
274 "__llvm_profile_register_functions", M);
275 RegisterF->setUnnamedAddr(true);
276 if (Options.NoRedZone)
277 RegisterF->addFnAttr(Attribute::NoRedZone);
279 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
280 auto *RuntimeRegisterF =
281 Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
282 "__llvm_profile_register_function", M);
284 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
285 for (Value *Data : UsedVars)
286 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
290 void InstrProfiling::emitRuntimeHook() {
291 const char *const RuntimeVarName = "__llvm_profile_runtime";
292 const char *const RuntimeUserName = "__llvm_profile_runtime_user";
294 // If the module's provided its own runtime, we don't need to do anything.
295 if (M->getGlobalVariable(RuntimeVarName))
298 // Declare an external variable that will pull in the runtime initialization.
299 auto *Int32Ty = Type::getInt32Ty(M->getContext());
301 new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
302 nullptr, RuntimeVarName);
304 // Make a function that uses it.
306 Function::Create(FunctionType::get(Int32Ty, false),
307 GlobalValue::LinkOnceODRLinkage, RuntimeUserName, M);
308 User->addFnAttr(Attribute::NoInline);
309 if (Options.NoRedZone)
310 User->addFnAttr(Attribute::NoRedZone);
311 User->setVisibility(GlobalValue::HiddenVisibility);
313 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
314 auto *Load = IRB.CreateLoad(Var);
317 // Mark the user variable as used so that it isn't stripped out.
318 UsedVars.push_back(User);
321 void InstrProfiling::emitUses() {
322 if (UsedVars.empty())
325 GlobalVariable *LLVMUsed = M->getGlobalVariable("llvm.used");
326 std::vector<Constant *> MergedVars;
328 // Collect the existing members of llvm.used.
329 ConstantArray *Inits = cast<ConstantArray>(LLVMUsed->getInitializer());
330 for (unsigned I = 0, E = Inits->getNumOperands(); I != E; ++I)
331 MergedVars.push_back(Inits->getOperand(I));
332 LLVMUsed->eraseFromParent();
335 Type *i8PTy = Type::getInt8PtrTy(M->getContext());
336 // Add uses for our data.
337 for (auto *Value : UsedVars)
338 MergedVars.push_back(
339 ConstantExpr::getBitCast(cast<Constant>(Value), i8PTy));
341 // Recreate llvm.used.
342 ArrayType *ATy = ArrayType::get(i8PTy, MergedVars.size());
344 new GlobalVariable(*M, ATy, false, GlobalValue::AppendingLinkage,
345 ConstantArray::get(ATy, MergedVars), "llvm.used");
347 LLVMUsed->setSection("llvm.metadata");
350 void InstrProfiling::emitInitialization() {
351 std::string InstrProfileOutput = Options.InstrProfileOutput;
353 Constant *RegisterF = M->getFunction("__llvm_profile_register_functions");
354 if (!RegisterF && InstrProfileOutput.empty())
357 // Create the initialization function.
358 auto *VoidTy = Type::getVoidTy(M->getContext());
360 Function::Create(FunctionType::get(VoidTy, false),
361 GlobalValue::InternalLinkage, "__llvm_profile_init", M);
362 F->setUnnamedAddr(true);
363 F->addFnAttr(Attribute::NoInline);
364 if (Options.NoRedZone)
365 F->addFnAttr(Attribute::NoRedZone);
367 // Add the basic block and the necessary calls.
368 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
370 IRB.CreateCall(RegisterF, {});
371 if (!InstrProfileOutput.empty()) {
372 auto *Int8PtrTy = Type::getInt8PtrTy(M->getContext());
373 auto *SetNameTy = FunctionType::get(VoidTy, Int8PtrTy, false);
375 Function::Create(SetNameTy, GlobalValue::ExternalLinkage,
376 "__llvm_profile_override_default_filename", M);
378 // Create variable for profile name.
379 Constant *ProfileNameConst =
380 ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
381 GlobalVariable *ProfileName =
382 new GlobalVariable(*M, ProfileNameConst->getType(), true,
383 GlobalValue::PrivateLinkage, ProfileNameConst);
385 IRB.CreateCall(SetNameF, IRB.CreatePointerCast(ProfileName, Int8PtrTy));
389 appendToGlobalCtors(*M, F, 0);