Add a division operator to BlockFrequency.

Allow a BlockFrequency to be divided by a non-zero BranchProbability
with saturating arithmetic. This will be used to compute the frequency
of a loop header given the probability of leaving the loop.

Our long division algorithm already saturates on overflow, so that was a
freebie.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@185184 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/llvm/Support/BlockFrequency.h b/include/llvm/Support/BlockFrequency.h
index 85e9437a571679e49d554fa63d523a1a48e80147..147d52abe418ceb089132f18ac60b9c174284bcb 100644 (file)
--- a/include/llvm/Support/BlockFrequency.h
+++ b/include/llvm/Support/BlockFrequency.h
@@ -27,6 +27,9 @@ class BlockFrequency {
   uint64_t Frequency;
   static const int64_t ENTRY_FREQ = 1 << 14;
 
+  // Scale frequency by N/D, saturating on overflow.
+  void scale(uint32_t N, uint32_t D);
+
 public:
   BlockFrequency(uint64_t Freq = 0) : Frequency(Freq) { }
 
@@ -42,6 +45,11 @@ public:
   BlockFrequency &operator*=(const BranchProbability &Prob);
   const BlockFrequency operator*(const BranchProbability &Prob) const;
 
+  /// \brief Divide by a non-zero branch probability using saturating
+  /// arithmetic.
+  BlockFrequency &operator/=(const BranchProbability &Prob);
+  BlockFrequency operator/(const BranchProbability &Prob) const;
+
   /// \brief Adds another block frequency using saturating arithmetic.
   BlockFrequency &operator+=(const BlockFrequency &Freq);
   const BlockFrequency operator+(const BlockFrequency &Freq) const;

diff --git a/lib/Support/BlockFrequency.cpp b/lib/Support/BlockFrequency.cpp
index 572dbf576548fe52fe15e996c321ff6f19b51110..08fa620eb8839ce93e258a40642f5dc18c0b5d44 100644 (file)
--- a/lib/Support/BlockFrequency.cpp
+++ b/lib/Support/BlockFrequency.cpp
@@ -42,12 +42,14 @@ void mult96bit(uint64_t freq, uint32_t N, uint64_t W[2]) {
 }
 
 
-/// div96bit - Divide 96-bit value stored in W array by D. Return 64-bit frequency.
+/// div96bit - Divide 96-bit value stored in W array by D.
+/// Return 64-bit quotient, saturated to UINT64_MAX on overflow.
 uint64_t div96bit(uint64_t W[2], uint32_t D) {
   uint64_t y = W[0];
   uint64_t x = W[1];
   int i;
 
+  // This long division algorithm automatically saturates on overflow.
   for (i = 1; i <= 64 && x; ++i) {
     uint32_t t = (int)x >> 31;
     x = (x << 1) | (y >> 63);
@@ -63,31 +65,30 @@ uint64_t div96bit(uint64_t W[2], uint32_t D) {
 
 }
 
+void BlockFrequency::scale(uint32_t N, uint32_t D) {
+  assert(D != 0 && "Division by zero");
 
-BlockFrequency &BlockFrequency::operator*=(const BranchProbability &Prob) {
-  uint32_t n = Prob.getNumerator();
-  uint32_t d = Prob.getDenominator();
-
-  assert(n <= d && "Probability must be less or equal to 1.");
-
-  // Calculate Frequency * n.
-  uint64_t mulLo = (Frequency & UINT32_MAX) * n;
-  uint64_t mulHi = (Frequency >> 32) * n;
-  uint64_t mulRes = (mulHi << 32) + mulLo;
-
-  // If there was overflow use 96-bit operations.
-  if (mulHi > UINT32_MAX || mulRes < mulLo) {
-    // 96-bit value represented as W[1]:W[0].
-    uint64_t W[2];
-
-    // Probability is less or equal to 1 which means that results must fit
-    // 64-bit.
-    mult96bit(Frequency, n, W);
-    Frequency = div96bit(W, d);
-    return *this;
+  // Calculate Frequency * N.
+  uint64_t MulLo = (Frequency & UINT32_MAX) * N;
+  uint64_t MulHi = (Frequency >> 32) * N;
+  uint64_t MulRes = (MulHi << 32) + MulLo;
+
+  // If the product fits in 64 bits, just use built-in division.
+  if (MulHi <= UINT32_MAX && MulRes <= MulLo) {
+    Frequency = MulRes / D;
+    return;
   }
 
-  Frequency = mulRes / d;
+  // Product overflowed, use 96-bit operations.
+  // 96-bit value represented as W[1]:W[0].
+  uint64_t W[2];
+  mult96bit(Frequency, N, W);
+  Frequency = div96bit(W, D);
+  return;
+}
+
+BlockFrequency &BlockFrequency::operator*=(const BranchProbability &Prob) {
+  scale(Prob.getNumerator(), Prob.getDenominator());
   return *this;
 }
 
@@ -98,6 +99,17 @@ BlockFrequency::operator*(const BranchProbability &Prob) const {
   return Freq;
 }
 
+BlockFrequency &BlockFrequency::operator/=(const BranchProbability &Prob) {
+  scale(Prob.getDenominator(), Prob.getNumerator());
+  return *this;
+}
+
+BlockFrequency BlockFrequency::operator/(const BranchProbability &Prob) const {
+  BlockFrequency Freq(Frequency);
+  Freq /= Prob;
+  return Freq;
+}
+
 BlockFrequency &BlockFrequency::operator+=(const BlockFrequency &Freq) {
   uint64_t Before = Freq.Frequency;
   Frequency += Freq.Frequency;

diff --git a/unittests/Support/BlockFrequencyTest.cpp b/unittests/Support/BlockFrequencyTest.cpp
index ff66bc4e45aae13499a9ff5bac3412eeb50fc63d..4bcddfebd9aa6a105ea7d53e997aaa184b4e65ac 100644 (file)
--- a/unittests/Support/BlockFrequencyTest.cpp
+++ b/unittests/Support/BlockFrequencyTest.cpp
@@ -52,6 +52,24 @@ TEST(BlockFrequencyTest, MaxToMax) {
   EXPECT_EQ(Freq.getFrequency(), UINT64_MAX);
 }
 
+TEST(BlockFrequency, Divide) {
+  BlockFrequency Freq(0x3333333333333333ULL);
+  Freq /= BranchProbability(1, 2);
+  EXPECT_EQ(Freq.getFrequency(), 0x6666666666666666ULL);
+}
+
+TEST(BlockFrequencyTest, Saturate) {
+  BlockFrequency Freq(0x3333333333333333ULL);
+  Freq /= BranchProbability(100, 300);
+  EXPECT_EQ(Freq.getFrequency(), 0x9999999999999999ULL);
+  Freq /= BranchProbability(1, 2);
+  EXPECT_EQ(Freq.getFrequency(), UINT64_MAX);
+
+  Freq = 0x1000000000000000ULL;
+  Freq /= BranchProbability(10000, 160000);
+  EXPECT_EQ(Freq.getFrequency(), UINT64_MAX);
+}
+
 TEST(BlockFrequencyTest, ProbabilityCompare) {
   BranchProbability A(4, 5);
   BranchProbability B(4U << 29, 5U << 29);