Add hardware crc impl

[folly.git] / folly / Checksum.cpp

diff --git a/folly/Checksum.cpp b/folly/Checksum.cpp
index 6ec07893265288a41cd314ddfc61187932a72e71..cbc99913c8428ee6a6fdd5ec2a7e50a73aa5a2f6 100644 (file)
--- a/folly/Checksum.cpp
+++ b/folly/Checksum.cpp
@@ -15,10 +15,11 @@
  */
 
 #include <folly/Checksum.h>
-#include <algorithm>
-#include <stdexcept>
 #include <boost/crc.hpp>
 #include <folly/CpuId.h>
+#include <folly/detail/ChecksumDetail.h>
+#include <algorithm>
+#include <stdexcept>
 
 #if FOLLY_X64 && (__SSE4_2__ || defined(__clang__) || __GNUC_PREREQ(4, 9))
 #include <nmmintrin.h>
@@ -28,6 +29,8 @@ namespace folly {
 
 namespace detail {
 
+uint32_t
+crc32c_sw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum);
 #if FOLLY_X64 && (__SSE4_2__ || defined(__clang__) || __GNUC_PREREQ(4, 9))
 
 // Fast SIMD implementation of CRC-32C for x86 with SSE 4.2
@@ -64,11 +67,43 @@ uint32_t crc32c_hw(const uint8_t *data, size_t nbytes,
   return sum;
 }
 
+uint32_t
+crc32_sw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum);
+
+// Fast SIMD implementation of CRC-32 for x86 with pclmul
+uint32_t
+crc32_hw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
+  uint32_t sum = startingChecksum;
+  size_t offset = 0;
+
+  // Process unaligned bytes
+  if ((uintptr_t)data & 15) {
+    size_t limit = std::min(nbytes, -(uintptr_t)data & 15);
+    sum = crc32_sw(data, limit, sum);
+    offset += limit;
+    nbytes -= limit;
+  }
+
+  if (nbytes >= 16) {
+    sum = crc32_hw_aligned(sum, (const __m128i*)(data + offset), nbytes / 16);
+    offset += nbytes & ~15;
+    nbytes &= 15;
+  }
+
+  // Remaining unaligned bytes
+  return crc32_sw(data + offset, nbytes, sum);
+}
+
 bool crc32c_hw_supported() {
   static folly::CpuId id;
   return id.sse42();
 }
 
+bool crc32_hw_supported() {
+  static folly::CpuId id;
+  return id.sse42();
+}
+
 #else
 
 uint32_t crc32c_hw(const uint8_t *data, size_t nbytes,
@@ -80,11 +115,13 @@ bool crc32c_hw_supported() {
   return false;
 }
 
+bool crc32_hw_supported() {
+  return false;
+}
 #endif
 
-uint32_t crc32c_sw(const uint8_t *data, size_t nbytes,
-    uint32_t startingChecksum) {
-
+template <uint32_t CRC_POLYNOMIAL>
+uint32_t crc_sw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
   // Reverse the bits in the starting checksum so they'll be in the
   // right internal format for Boost's CRC engine.
   //     O(1)-time, branchless bit reversal algorithm from
@@ -100,13 +137,24 @@ uint32_t crc32c_sw(const uint8_t *data, size_t nbytes,
   startingChecksum = (startingChecksum >> 16) |
       (startingChecksum << 16);
 
-  static const uint32_t CRC32C_POLYNOMIAL = 0x1EDC6F41;
-  boost::crc_optimal<32, CRC32C_POLYNOMIAL, ~0U, 0, true, true> sum(
+  boost::crc_optimal<32, CRC_POLYNOMIAL, ~0U, 0, true, true> sum(
       startingChecksum);
   sum.process_bytes(data, nbytes);
   return sum.checksum();
 }
 
+uint32_t
+crc32c_sw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
+  constexpr uint32_t CRC32C_POLYNOMIAL = 0x1EDC6F41;
+  return crc_sw<CRC32C_POLYNOMIAL>(data, nbytes, startingChecksum);
+}
+
+uint32_t
+crc32_sw(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
+  constexpr uint32_t CRC32_POLYNOMIAL = 0x04C11DB7;
+  return crc_sw<CRC32_POLYNOMIAL>(data, nbytes, startingChecksum);
+}
+
 } // folly::detail
 
 uint32_t crc32c(const uint8_t *data, size_t nbytes,
@@ -118,4 +166,12 @@ uint32_t crc32c(const uint8_t *data, size_t nbytes,
   }
 }
 
+uint32_t crc32(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
+  if (detail::crc32_hw_supported()) {
+    return detail::crc32_hw(data, nbytes, startingChecksum);
+  } else {
+    return detail::crc32_sw(data, nbytes, startingChecksum);
+  }
+}
+
 } // folly