+ setEdgeProbability(BB, i, {Weights[i], static_cast<uint32_t>(WeightSum)});
+
+ assert(WeightSum <= UINT32_MAX &&
+ "Expected weights to scale down to 32 bits");
+
+ return true;
+}
+
+/// \brief Calculate edge weights for edges leading to cold blocks.
+///
+/// A cold block is one post-dominated by a block with a call to a
+/// cold function. Those edges are unlikely to be taken, so we give
+/// them relatively low weight.
+///
+/// Return true if we could compute the weights for cold edges.
+/// Return false, otherwise.
+bool BranchProbabilityInfo::calcColdCallHeuristics(BasicBlock *BB) {
+ TerminatorInst *TI = BB->getTerminator();
+ if (TI->getNumSuccessors() == 0)
+ return false;
+
+ // Determine which successors are post-dominated by a cold block.
+ SmallVector<unsigned, 4> ColdEdges;
+ SmallVector<unsigned, 4> NormalEdges;
+ for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I)
+ if (PostDominatedByColdCall.count(*I))
+ ColdEdges.push_back(I.getSuccessorIndex());
+ else
+ NormalEdges.push_back(I.getSuccessorIndex());
+
+ // If all successors are in the set of blocks post-dominated by cold calls,
+ // this block is in the set post-dominated by cold calls.
+ if (ColdEdges.size() == TI->getNumSuccessors())
+ PostDominatedByColdCall.insert(BB);
+ else {
+ // Otherwise, if the block itself contains a cold function, add it to the
+ // set of blocks postdominated by a cold call.
+ assert(!PostDominatedByColdCall.count(BB));
+ for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
+ if (CallInst *CI = dyn_cast<CallInst>(I))
+ if (CI->hasFnAttr(Attribute::Cold)) {
+ PostDominatedByColdCall.insert(BB);
+ break;
+ }
+ }
+
+ // Skip probabilities if this block has a single successor.
+ if (TI->getNumSuccessors() == 1 || ColdEdges.empty())
+ return false;
+
+ if (NormalEdges.empty()) {
+ BranchProbability Prob(1, ColdEdges.size());
+ for (unsigned SuccIdx : ColdEdges)
+ setEdgeProbability(BB, SuccIdx, Prob);
+ return true;
+ }
+
+ BranchProbability ColdProb(CC_TAKEN_WEIGHT,
+ (CC_TAKEN_WEIGHT + CC_NONTAKEN_WEIGHT) *
+ ColdEdges.size());
+ BranchProbability NormalProb(CC_NONTAKEN_WEIGHT,
+ (CC_TAKEN_WEIGHT + CC_NONTAKEN_WEIGHT) *
+ NormalEdges.size());
+
+ for (unsigned SuccIdx : ColdEdges)
+ setEdgeProbability(BB, SuccIdx, ColdProb);
+ for (unsigned SuccIdx : NormalEdges)
+ setEdgeProbability(BB, SuccIdx, NormalProb);