From db66e2f4b93dee8c8834dede216e068a3a3e40af Mon Sep 17 00:00:00 2001 From: Brian Pane Date: Mon, 7 Jul 2014 10:27:30 -0700 Subject: [PATCH] Move the branchless, parallel ASCII toLower into folly Summary: * Moved the fast toLower code into folly * Updated the name to emphasize that it's ASCII-only Test Plan: * Unit tests and benchmarks included. Reviewed By: tudorb@fb.com Subscribers: ruibalp, bmatheny FB internal diff: D1421056 --- folly/String.cpp | 140 +++++++++++++++++++++++++++++++++ folly/String.h | 13 +++ folly/test/StringBenchmark.cpp | 67 ++++++++++++++++ folly/test/StringTest.cpp | 43 ++++++++++ 4 files changed, 263 insertions(+) create mode 100644 folly/test/StringBenchmark.cpp diff --git a/folly/String.cpp b/folly/String.cpp index 612db8cb..f6fc8def 100644 --- a/folly/String.cpp +++ b/folly/String.cpp @@ -348,6 +348,146 @@ loop: return sp; } +namespace { + +void toLowerAscii8(char& c) { + // Branchless tolower, based on the input-rotating trick described + // at http://www.azillionmonkeys.com/qed/asmexample.html + // + // This algorithm depends on an observation: each uppercase + // ASCII character can be converted to its lowercase equivalent + // by adding 0x20. + + // Step 1: Clear the high order bit. We'll deal with it in Step 5. + unsigned char rotated = c & 0x7f; + // Currently, the value of rotated, as a function of the original c is: + // below 'A': 0- 64 + // 'A'-'Z': 65- 90 + // above 'Z': 91-127 + + // Step 2: Add 0x25 (37) + rotated += 0x25; + // Now the value of rotated, as a function of the original c is: + // below 'A': 37-101 + // 'A'-'Z': 102-127 + // above 'Z': 128-164 + + // Step 3: clear the high order bit + rotated &= 0x7f; + // below 'A': 37-101 + // 'A'-'Z': 102-127 + // above 'Z': 0- 36 + + // Step 4: Add 0x1a (26) + rotated += 0x1a; + // below 'A': 63-127 + // 'A'-'Z': 128-153 + // above 'Z': 25- 62 + + // At this point, note that only the uppercase letters have been + // transformed into values with the high order bit set (128 and above). + + // Step 5: Shift the high order bit 2 spaces to the right: the spot + // where the only 1 bit in 0x20 is. But first, how we ignored the + // high order bit of the original c in step 1? If that bit was set, + // we may have just gotten a false match on a value in the range + // 128+'A' to 128+'Z'. To correct this, need to clear the high order + // bit of rotated if the high order bit of c is set. Since we don't + // care about the other bits in rotated, the easiest thing to do + // is invert all the bits in c and bitwise-and them with rotated. + rotated &= ~c; + rotated >>= 2; + + // Step 6: Apply a mask to clear everything except the 0x20 bit + // in rotated. + rotated &= 0x20; + + // At this point, rotated is 0x20 if c is 'A'-'Z' and 0x00 otherwise + + // Step 7: Add rotated to c + c += rotated; +} + +void toLowerAscii32(uint32_t& c) { + // Besides being branchless, the algorithm in toLowerAscii8() has another + // interesting property: None of the addition operations will cause + // an overflow in the 8-bit value. So we can pack four 8-bit values + // into a uint32_t and run each operation on all four values in parallel + // without having to use any CPU-specific SIMD instructions. + uint32_t rotated = c & uint32_t(0x7f7f7f7fL); + rotated += uint32_t(0x25252525L); + rotated &= uint32_t(0x7f7f7f7fL); + rotated += uint32_t(0x1a1a1a1aL); + + // Step 5 involves a shift, so some bits will spill over from each + // 8-bit value into the next. But that's okay, because they're bits + // that will be cleared by the mask in step 6 anyway. + rotated &= ~c; + rotated >>= 2; + rotated &= uint32_t(0x20202020L); + c += rotated; +} + +void toLowerAscii64(uint64_t& c) { + // 64-bit version of toLower32 + uint64_t rotated = c & uint64_t(0x7f7f7f7f7f7f7f7fL); + rotated += uint64_t(0x2525252525252525L); + rotated &= uint64_t(0x7f7f7f7f7f7f7f7fL); + rotated += uint64_t(0x1a1a1a1a1a1a1a1aL); + rotated &= ~c; + rotated >>= 2; + rotated &= uint64_t(0x2020202020202020L); + c += rotated; +} + +} // anon namespace + +void toLowerAscii(char* str, size_t length) { + static const size_t kAlignMask64 = 7; + static const size_t kAlignMask32 = 3; + + // Convert a character at a time until we reach an address that + // is at least 32-bit aligned + size_t n = (size_t)str; + n &= kAlignMask32; + n = std::min(n, length); + size_t offset = 0; + if (n != 0) { + do { + toLowerAscii8(str[offset]); + offset++; + } while (offset < n); + } + + n = (size_t)(str + offset); + n &= kAlignMask64; + if ((n != 0) && (offset + 4 < length)) { + // The next address is 32-bit aligned but not 64-bit aligned. + // Convert the next 4 bytes in order to get to the 64-bit aligned + // part of the input. + toLowerAscii32(*(uint32_t*)(str + offset)); + offset += 4; + } + + // Convert 8 characters at a time + while (offset + 8 < length) { + toLowerAscii64(*(uint64_t*)(str + offset)); + offset += 8; + } + + // Convert 4 characters at a time + while (offset + 4 < length) { + toLowerAscii32(*(uint32_t*)(str + offset)); + offset += 4; + } + + // Convert any characters remaining after the last 4-byte aligned group + while (offset < length) { + toLowerAscii8(str[offset]); + offset++; + } +} + namespace detail { size_t hexDumpLine(const void* ptr, size_t offset, size_t size, diff --git a/folly/String.h b/folly/String.h index 9a44f545..563f5313 100644 --- a/folly/String.h +++ b/folly/String.h @@ -509,6 +509,19 @@ std::string join(const Delim& delimiter, */ StringPiece skipWhitespace(StringPiece sp); +/** + * Fast, in-place lowercasing of ASCII alphabetic characters in strings. + * Leaves all other characters unchanged, including those with the 0x80 + * bit set. + * @param str String to convert + * @param len Length of str, in bytes + */ +void toLowerAscii(char* str, size_t length); + +inline void toLowerAscii(MutableStringPiece str) { + toLowerAscii(str.begin(), str.size()); +} + } // namespace folly // Hash functions to make std::string usable with e.g. hash_map diff --git a/folly/test/StringBenchmark.cpp b/folly/test/StringBenchmark.cpp new file mode 100644 index 00000000..e2566c66 --- /dev/null +++ b/folly/test/StringBenchmark.cpp @@ -0,0 +1,67 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +BENCHMARK(libc_tolower, iters) { + static const size_t kSize = 256; + // This array is static to keep the compiler from optimizing the + // entire function down to a no-op if it has an inlined impl of + // tolower and thus is able to tell that there are no side-effects. + // No side-effects + no writes to anything other than local variables + // + no return value = no need to run any of the code in the function. + // gcc, for example, makes that optimization with -O2. + static char input[kSize]; + for (size_t i = 0; i < kSize; i++) { + input[i] = (char)(i & 0xff); + } + for (auto i = iters; i > 0; i--) { + for (size_t offset = 0; offset < kSize; offset++) { + input[offset] = tolower(input[offset]); + } + } +} + +BENCHMARK(folly_toLowerAscii, iters) { + static const size_t kSize = 256; + static char input[kSize]; + for (size_t i = 0; i < kSize; i++) { + input[i] = (char)(i & 0xff); + } + for (auto i = iters; i > 0; i--) { + folly::toLowerAscii(input, kSize); + } +} + +int main(int argc, char** argv) { + google::ParseCommandLineFlags(&argc, &argv, true); + folly::runBenchmarks(); + if (FLAGS_benchmark) { + folly::runBenchmarks(); + } + return 0; +} + +/* +Results on x86_64: +============================================================================ +folly/test/StringBenchmark.cpp relative time/iter iters/s +============================================================================ +libc_tolower 1.30us 767.50K +folly_toLowerAscii 115.21ns 8.68M +============================================================================ +*/ diff --git a/folly/test/StringTest.cpp b/folly/test/StringTest.cpp index 0a1ac9e4..2e5e6922 100644 --- a/folly/test/StringTest.cpp +++ b/folly/test/StringTest.cpp @@ -1034,6 +1034,49 @@ TEST(String, humanify) { EXPECT_EQ("0x61ffffffffff", humanify(string("a\xff\xff\xff\xff\xff"))); } +namespace { + +void testToLowerAscii(Range src) { + char control[src.size()]; + memcpy(control, src.begin(), src.size()); + char test[src.size()]; + memcpy(test, src.begin(), src.size()); + for (size_t i = 0; i < src.size(); i++) { + control[i] = tolower(control[i]); + } + toLowerAscii(test, src.size()); + for (size_t i = 0; i < src.size(); i++) { + EXPECT_EQ(control[i], test[i]); + } +} + +} // anon namespace + +TEST(String, toLowerAsciiAligned) { + static const size_t kSize = 256; + char input[kSize]; + for (size_t i = 0; i < kSize; i++) { + input[i] = (char)(i & 0xff); + } + testToLowerAscii(Range(input, kSize)); +} + +TEST(String, toLowerAsciiUnaligned) { + static const size_t kSize = 256; + char input[kSize]; + for (size_t i = 0; i < kSize; i++) { + input[i] = (char)(i & 0xff); + } + // Test input buffers of several lengths to exercise all the + // cases: buffer at the start/middle/end of an aligned block, plus + // buffers that span multiple aligned blocks. + for (size_t length = 1; length < 11; length++) { + for (size_t offset = 0; offset + length <= kSize; offset++) { + testToLowerAscii(Range(input + offset, length)); + } + } +} + ////////////////////////////////////////////////////////////////////// BENCHMARK(splitOnSingleChar, iters) { -- 2.34.1