#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/InlineCost.h"
-#include "llvm/DataLayout.h"
-#include "llvm/Instructions.h"
-#include "llvm/IntrinsicInst.h"
-#include "llvm/Module.h"
-#include "llvm/Support/CallSite.h"
+#include "llvm/IR/CallSite.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Module.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
HintThreshold("inlinehint-threshold", cl::Hidden, cl::init(325),
cl::desc("Threshold for inlining functions with inline hint"));
+// We instroduce this threshold to help performance of instrumentation based
+// PGO before we actually hook up inliner with analysis passes such as BPI and
+// BFI.
+static cl::opt<int>
+ColdThreshold("inlinecold-threshold", cl::Hidden, cl::init(225),
+ cl::desc("Threshold for inlining functions with cold attribute"));
+
// Threshold to use when optsize is specified (and there is no -inline-limit).
const int OptSizeThreshold = 75;
/// getAnalysisUsage - For this class, we declare that we require and preserve
/// the call graph. If the derived class implements this method, it should
/// always explicitly call the implementation here.
-void Inliner::getAnalysisUsage(AnalysisUsage &Info) const {
- CallGraphSCCPass::getAnalysisUsage(Info);
+void Inliner::getAnalysisUsage(AnalysisUsage &AU) const {
+ CallGraphSCCPass::getAnalysisUsage(AU);
}
typedef DenseMap<ArrayType*, std::vector<AllocaInst*> >
InlinedArrayAllocasTy;
+/// \brief If the inlined function had a higher stack protection level than the
+/// calling function, then bump up the caller's stack protection level.
+static void AdjustCallerSSPLevel(Function *Caller, Function *Callee) {
+ // If upgrading the SSP attribute, clear out the old SSP Attributes first.
+ // Having multiple SSP attributes doesn't actually hurt, but it adds useless
+ // clutter to the IR.
+ AttrBuilder B;
+ B.addAttribute(Attribute::StackProtect)
+ .addAttribute(Attribute::StackProtectStrong);
+ AttributeSet OldSSPAttr = AttributeSet::get(Caller->getContext(),
+ AttributeSet::FunctionIndex,
+ B);
+ AttributeSet CallerAttr = Caller->getAttributes(),
+ CalleeAttr = Callee->getAttributes();
+
+ if (CalleeAttr.hasAttribute(AttributeSet::FunctionIndex,
+ Attribute::StackProtectReq)) {
+ Caller->removeAttributes(AttributeSet::FunctionIndex, OldSSPAttr);
+ Caller->addFnAttr(Attribute::StackProtectReq);
+ } else if (CalleeAttr.hasAttribute(AttributeSet::FunctionIndex,
+ Attribute::StackProtectStrong) &&
+ !CallerAttr.hasAttribute(AttributeSet::FunctionIndex,
+ Attribute::StackProtectReq)) {
+ Caller->removeAttributes(AttributeSet::FunctionIndex, OldSSPAttr);
+ Caller->addFnAttr(Attribute::StackProtectStrong);
+ } else if (CalleeAttr.hasAttribute(AttributeSet::FunctionIndex,
+ Attribute::StackProtect) &&
+ !CallerAttr.hasAttribute(AttributeSet::FunctionIndex,
+ Attribute::StackProtectReq) &&
+ !CallerAttr.hasAttribute(AttributeSet::FunctionIndex,
+ Attribute::StackProtectStrong))
+ Caller->addFnAttr(Attribute::StackProtect);
+}
+
/// InlineCallIfPossible - If it is possible to inline the specified call site,
/// do so and update the CallGraph for this operation.
///
/// any new allocas to the set if not possible.
static bool InlineCallIfPossible(CallSite CS, InlineFunctionInfo &IFI,
InlinedArrayAllocasTy &InlinedArrayAllocas,
- int InlineHistory, bool InsertLifetime) {
+ int InlineHistory, bool InsertLifetime,
+ const DataLayout *DL) {
Function *Callee = CS.getCalledFunction();
Function *Caller = CS.getCaller();
if (!InlineFunction(CS, IFI, InsertLifetime))
return false;
- // If the inlined function had a higher stack protection level than the
- // calling function, then bump up the caller's stack protection level.
- if (Callee->getFnAttributes().hasAttribute(Attributes::StackProtectReq))
- Caller->addFnAttr(Attributes::StackProtectReq);
- else if (Callee->getFnAttributes().hasAttribute(Attributes::StackProtect) &&
- !Caller->getFnAttributes().hasAttribute(Attributes::StackProtectReq))
- Caller->addFnAttr(Attributes::StackProtect);
+ AdjustCallerSSPLevel(Caller, Callee);
// Look at all of the allocas that we inlined through this call site. If we
// have already inlined other allocas through other calls into this function,
bool MergedAwayAlloca = false;
for (unsigned i = 0, e = AllocasForType.size(); i != e; ++i) {
AllocaInst *AvailableAlloca = AllocasForType[i];
+
+ unsigned Align1 = AI->getAlignment(),
+ Align2 = AvailableAlloca->getAlignment();
+ // If we don't have data layout information, and only one alloca is using
+ // the target default, then we can't safely merge them because we can't
+ // pick the greater alignment.
+ if (!DL && (!Align1 || !Align2) && Align1 != Align2)
+ continue;
// The available alloca has to be in the right function, not in some other
// function in this SCC.
<< *AvailableAlloca << '\n');
AI->replaceAllUsesWith(AvailableAlloca);
+
+ if (Align1 != Align2) {
+ if (!Align1 || !Align2) {
+ assert(DL && "DataLayout required to compare default alignments");
+ unsigned TypeAlign = DL->getABITypeAlignment(AI->getAllocatedType());
+
+ Align1 = Align1 ? Align1 : TypeAlign;
+ Align2 = Align2 ? Align2 : TypeAlign;
+ }
+
+ if (Align1 > Align2)
+ AvailableAlloca->setAlignment(AI->getAlignment());
+ }
+
AI->eraseFromParent();
MergedAwayAlloca = true;
++NumMergedAllocas;
// would decrease the threshold.
Function *Caller = CS.getCaller();
bool OptSize = Caller && !Caller->isDeclaration() &&
- Caller->getFnAttributes().hasAttribute(Attributes::OptimizeForSize);
+ Caller->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
+ Attribute::OptimizeForSize);
if (!(InlineLimit.getNumOccurrences() > 0) && OptSize &&
OptSizeThreshold < thres)
thres = OptSizeThreshold;
- // Listen to the inlinehint attribute when it would increase the threshold.
+ // Listen to the inlinehint attribute when it would increase the threshold
+ // and the caller does not need to minimize its size.
Function *Callee = CS.getCalledFunction();
bool InlineHint = Callee && !Callee->isDeclaration() &&
- Callee->getFnAttributes().hasAttribute(Attributes::InlineHint);
- if (InlineHint && HintThreshold > thres)
+ Callee->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
+ Attribute::InlineHint);
+ if (InlineHint && HintThreshold > thres
+ && !Caller->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
+ Attribute::MinSize))
thres = HintThreshold;
+ // Listen to the cold attribute when it would decrease the threshold.
+ bool ColdCallee = Callee && !Callee->isDeclaration() &&
+ Callee->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
+ Attribute::Cold);
+ if (ColdCallee && ColdThreshold < thres)
+ thres = ColdThreshold;
+
return thres;
}
bool callerWillBeRemoved = Caller->hasLocalLinkage();
// This bool tracks what happens if we DO inline C into B.
bool inliningPreventsSomeOuterInline = false;
- for (Value::use_iterator I = Caller->use_begin(), E =Caller->use_end();
- I != E; ++I) {
- CallSite CS2(*I);
+ for (User *U : Caller->users()) {
+ CallSite CS2(U);
// If this isn't a call to Caller (it could be some other sort
// of reference) skip it. Such references will prevent the caller
// one is set very low by getInlineCost, in anticipation that Caller will
// be removed entirely. We did not account for this above unless there
// is only one caller of Caller.
- if (callerWillBeRemoved && Caller->use_begin() != Caller->use_end())
+ if (callerWillBeRemoved && !Caller->use_empty())
TotalSecondaryCost += InlineConstants::LastCallToStaticBonus;
if (inliningPreventsSomeOuterInline && TotalSecondaryCost < IC.getCost()) {
}
bool Inliner::runOnSCC(CallGraphSCC &SCC) {
- CallGraph &CG = getAnalysis<CallGraph>();
- const DataLayout *TD = getAnalysisIfAvailable<DataLayout>();
+ CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
+ DataLayoutPass *DLP = getAnalysisIfAvailable<DataLayoutPass>();
+ const DataLayout *DL = DLP ? &DLP->getDataLayout() : 0;
const TargetLibraryInfo *TLI = getAnalysisIfAvailable<TargetLibraryInfo>();
SmallPtrSet<Function*, 8> SCCFunctions;
InlinedArrayAllocasTy InlinedArrayAllocas;
- InlineFunctionInfo InlineInfo(&CG, TD);
+ InlineFunctionInfo InlineInfo(&CG, DL);
// Now that we have all of the call sites, loop over them and inline them if
// it looks profitable to do so.
if (!shouldInline(CS))
continue;
+ // Get DebugLoc to report. CS will be invalid after Inliner.
+ DebugLoc DLoc = CS.getInstruction()->getDebugLoc();
+
// Attempt to inline the function.
if (!InlineCallIfPossible(CS, InlineInfo, InlinedArrayAllocas,
- InlineHistoryID, InsertLifetime))
+ InlineHistoryID, InsertLifetime, DL))
continue;
++NumInlined;
-
+
+ // Report the inline decision.
+ Caller->getContext().emitOptimizationRemark(
+ DEBUG_TYPE, *Caller, DLoc,
+ Twine(Callee->getName() + " inlined into " + Caller->getName()));
+
// If inlining this function gave us any new call sites, throw them
// onto our worklist to process. They are useful inline candidates.
if (!InlineInfo.InlinedCalls.empty()) {
// about always-inline functions. This is a bit of a hack to share code
// between here and the InlineAlways pass.
if (AlwaysInlineOnly &&
- !F->getFnAttributes().hasAttribute(Attributes::AlwaysInline))
+ !F->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
+ Attribute::AlwaysInline))
continue;
// If the only remaining users of the function are dead constants, remove
projects / oota-llvm.git / blobdiff