From: Tudor Bosman Date: Thu, 6 Feb 2014 04:13:40 +0000 (-0800) Subject: Cache open ELF files in Symbolizer X-Git-Tag: v0.22.0~701 X-Git-Url: http://plrg.eecs.uci.edu/git/?a=commitdiff_plain;h=7224b63e71e383e8ae78d2e118fbb0a13864eaee;p=folly.git Cache open ELF files in Symbolizer Summary: Rather than opening and closing Elf files every time we symbolize them, we open the first time and cache. We're using two caches: one for the signal handler (fixed size, slow, but async-signal-safe) and one for exception tracing and general use. Also, unrelated, removed two useless frames from the stack trace dump in the signal handler. Test Plan: tests added Reviewed By: lucian@fb.com FB internal diff: D1161444 --- diff --git a/folly/experimental/symbolizer/ElfCache.cpp b/folly/experimental/symbolizer/ElfCache.cpp new file mode 100644 index 00000000..79652a49 --- /dev/null +++ b/folly/experimental/symbolizer/ElfCache.cpp @@ -0,0 +1,97 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "folly/experimental/symbolizer/ElfCache.h" + +namespace folly { namespace symbolizer { + +SignalSafeElfCache::SignalSafeElfCache(size_t capacity) { + map_.reserve(capacity); + slots_.reserve(capacity); + + // Preallocate + for (size_t i = 0; i < capacity; ++i) { + slots_.push_back(std::make_shared()); + } +} + +std::shared_ptr SignalSafeElfCache::getFile(StringPiece p) { + if (p.size() > Path::kMaxSize) { + return nullptr; + } + + Path path(p); + auto pos = map_.find(path); + if (pos != map_.end()) { + return slots_[pos->second]; + } + + size_t n = map_.size(); + if (n >= slots_.size()) { + DCHECK_EQ(map_.size(), slots_.size()); + return nullptr; + } + + auto& f = slots_[n]; + if (f->openNoThrow(path.data()) == -1) { + return nullptr; + } + + map_[path] = n; + return f; +} + +ElfCache::ElfCache(size_t capacity) : capacity_(capacity) { } + +std::shared_ptr ElfCache::getFile(StringPiece p) { + auto path = p.str(); + + std::lock_guard lock(mutex_); + + auto pos = files_.find(path); + if (pos != files_.end()) { + // Found, move to back (MRU) + auto& entry = pos->second; + lruList_.erase(lruList_.iterator_to(*entry)); + lruList_.push_back(*entry); + return filePtr(entry); + } + + auto entry = std::make_shared(); + + // No negative caching + if (entry->file.openNoThrow(path.c_str()) == -1) { + return nullptr; + } + + if (files_.size() == capacity_) { + // Evict LRU + lruList_.pop_front(); + } + + files_.emplace(std::move(path), entry); + lruList_.push_back(*entry); + + return filePtr(entry); +} + +std::shared_ptr ElfCache::filePtr(const std::shared_ptr& e) { + // share ownership + return std::shared_ptr(e, &e->file); +} + +}} // namespaces + diff --git a/folly/experimental/symbolizer/ElfCache.h b/folly/experimental/symbolizer/ElfCache.h new file mode 100644 index 00000000..f6581940 --- /dev/null +++ b/folly/experimental/symbolizer/ElfCache.h @@ -0,0 +1,130 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FOLLY_SYMBOLIZER_ELFCACHE_H_ +#define FOLLY_SYMBOLIZER_ELFCACHE_H_ + +#include +#include // for PATH_MAX +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "folly/experimental/symbolizer/Elf.h" + +namespace folly { namespace symbolizer { + +class ElfCacheBase { + public: + virtual std::shared_ptr getFile(StringPiece path) = 0; + virtual ~ElfCacheBase() { } +}; + +/** + * Cache ELF files. Async-signal-safe: does memory allocation upfront. + * + * Will not grow; once the capacity is reached, lookups for files that + * aren't already in the cache will fail (return nullptr). + * + * Not MT-safe. May not be used concurrently from multiple threads. + * + * NOTE that async-signal-safety is preserved only as long as the + * SignalSafeElfCache object exists; after the SignalSafeElfCache object + * is destroyed, destroying returned shared_ptr objects may + * cause ElfFile objects to be destroyed, and that's not async-signal-safe. + */ +class SignalSafeElfCache : public ElfCacheBase { + public: + explicit SignalSafeElfCache(size_t capacity); + + std::shared_ptr getFile(StringPiece path) override; + + private: + // We can't use std::string (allocating memory is bad!) so we roll our + // own wrapper around a fixed-size, null-terminated string. + class Path : private boost::totally_ordered { + public: + explicit Path(StringPiece s) { + DCHECK_LE(s.size(), kMaxSize); + memcpy(data_, s.data(), s.size()); + data_[s.size()] = '\0'; + } + + bool operator<(const Path& other) const { + return strcmp(data_, other.data_) < 0; + } + + bool operator==(const Path& other) const { + return strcmp(data_, other.data_) == 0; + } + + const char* data() const { + return data_; + } + + static constexpr size_t kMaxSize = PATH_MAX - 1; + + private: + char data_[kMaxSize + 1]; + }; + + boost::container::flat_map map_; + std::vector> slots_; +}; + +/** + * General-purpose ELF file cache. + * + * LRU of given capacity. MT-safe (uses locking). Not async-signal-safe. + */ +class ElfCache : public ElfCacheBase { + public: + explicit ElfCache(size_t capacity); + + std::shared_ptr getFile(StringPiece path) override; + + private: + std::mutex mutex_; + + typedef boost::intrusive::list_member_hook<> LruLink; + struct Entry { + ElfFile file; + LruLink lruLink; + }; + + static std::shared_ptr filePtr(const std::shared_ptr& e); + + size_t capacity_; + std::unordered_map> files_; + + typedef boost::intrusive::list< + Entry, + boost::intrusive::member_hook, + boost::intrusive::constant_time_size> LruList; + LruList lruList_; +}; + +}} // namespaces + +#endif /* FOLLY_SYMBOLIZER_ELFCACHE_H_ */ + diff --git a/folly/experimental/symbolizer/SignalHandler.cpp b/folly/experimental/symbolizer/SignalHandler.cpp index 89a58d1c..5c3d56d9 100644 --- a/folly/experimental/symbolizer/SignalHandler.cpp +++ b/folly/experimental/symbolizer/SignalHandler.cpp @@ -188,6 +188,16 @@ void dumpSignalInfo(int signum, siginfo_t* siginfo) { print("), stack trace: ***\n"); } +namespace { +constexpr size_t kDefaultCapacity = 500; + +// Note: not thread-safe, but that's okay, as we only let one thread +// in our signal handler at a time. +SignalSafeElfCache signalSafeElfCache(kDefaultCapacity); +} // namespace + +void dumpStackTrace() __attribute__((noinline)); + void dumpStackTrace() { SCOPE_EXIT { fsyncNoInt(STDERR_FILENO); }; // Get and symbolize stack trace @@ -198,11 +208,17 @@ void dumpStackTrace() { if (!getStackTraceSafe(addresses)) { print("(error retrieving stack trace)\n"); } else { - Symbolizer symbolizer; + Symbolizer symbolizer(&signalSafeElfCache); symbolizer.symbolize(addresses); FDSymbolizePrinter printer(STDERR_FILENO, SymbolizePrinter::COLOR_IF_TTY); - printer.println(addresses); + + // Skip the top 2 frames: + // getStackTraceSafe + // dumpStackTrace (here) + // + // Leaving signalHandler on the stack for clarity, I think. + printer.println(addresses, 2); } } diff --git a/folly/experimental/symbolizer/Symbolizer.cpp b/folly/experimental/symbolizer/Symbolizer.cpp index d7f4db96..2498b7d9 100644 --- a/folly/experimental/symbolizer/Symbolizer.cpp +++ b/folly/experimental/symbolizer/Symbolizer.cpp @@ -152,8 +152,18 @@ bool parseProcMapsLine(StringPiece line, return true; } +ElfCache* defaultElfCache() { + static constexpr size_t defaultCapacity = 500; + static ElfCache cache(defaultCapacity); + return &cache; +} + } // namespace +Symbolizer::Symbolizer(ElfCacheBase* cache) + : cache_(cache ?: defaultElfCache()) { +} + void Symbolizer::symbolize(const uintptr_t* addresses, SymbolizedFrame* frames, size_t addressCount) { @@ -196,7 +206,7 @@ void Symbolizer::symbolize(const uintptr_t* addresses, } bool first = true; - ElfFile* elfFile = nullptr; + std::shared_ptr elfFile; // See if any addresses are here for (size_t i = 0; i < addressCount; ++i) { @@ -218,16 +228,7 @@ void Symbolizer::symbolize(const uintptr_t* addresses, // Open the file on first use if (first) { first = false; - if (fileCount_ < kMaxFiles && - !fileName.empty() && - fileName.size() < sizeof(fileNameBuf)) { - memcpy(fileNameBuf, fileName.data(), fileName.size()); - fileNameBuf[fileName.size()] = '\0'; - auto& f = files_[fileCount_++]; - if (f.openNoThrow(fileNameBuf) != -1) { - elfFile = &f; - } - } + elfFile = cache_->getFile(fileName); } if (!elfFile) { @@ -245,7 +246,7 @@ void Symbolizer::symbolize(const uintptr_t* addresses, frame.name = name; } - Dwarf(elfFile).findAddress(fileAddress, frame.location); + Dwarf(elfFile.get()).findAddress(fileAddress, frame.location); } } diff --git a/folly/experimental/symbolizer/Symbolizer.h b/folly/experimental/symbolizer/Symbolizer.h index c9341f4f..5595a53b 100644 --- a/folly/experimental/symbolizer/Symbolizer.h +++ b/folly/experimental/symbolizer/Symbolizer.h @@ -23,7 +23,9 @@ #include "folly/FBString.h" #include "folly/Range.h" +#include "folly/String.h" #include "folly/experimental/symbolizer/Elf.h" +#include "folly/experimental/symbolizer/ElfCache.h" #include "folly/experimental/symbolizer/Dwarf.h" #include "folly/experimental/symbolizer/StackTrace.h" @@ -42,6 +44,13 @@ struct SymbolizedFrame { bool found; StringPiece name; Dwarf::LocationInfo location; + + /** + * Demangle the name and return it. Not async-signal-safe; allocates memory. + */ + fbstring demangledName() const { + return demangle(name.fbstr().c_str()); + } }; template @@ -93,7 +102,7 @@ inline bool getStackTraceSafe(FrameArray& fa) { class Symbolizer { public: - Symbolizer() : fileCount_(0) { } + explicit Symbolizer(ElfCacheBase* cache = nullptr); /** * Symbolize given addresses. @@ -116,11 +125,7 @@ class Symbolizer { } private: - // We can't allocate memory, so we'll preallocate room. - // "1023 shared libraries should be enough for everyone" - static constexpr size_t kMaxFiles = 1024; - size_t fileCount_; - ElfFile files_[kMaxFiles]; + ElfCacheBase* cache_; }; /** diff --git a/folly/experimental/symbolizer/test/SymbolizerTest.cpp b/folly/experimental/symbolizer/test/SymbolizerTest.cpp index 1b790278..cd2ab95b 100644 --- a/folly/experimental/symbolizer/test/SymbolizerTest.cpp +++ b/folly/experimental/symbolizer/test/SymbolizerTest.cpp @@ -16,6 +16,8 @@ #include "folly/experimental/symbolizer/Symbolizer.h" +#include + #include #include "folly/Range.h" @@ -42,4 +44,68 @@ TEST(Symbolizer, Single) { EXPECT_EQ("SymbolizerTest.cpp", basename.str()); } +FrameArray<100> goldenFrames; + +int comparator(const void* ap, const void* bp) { + getStackTrace(goldenFrames); + + int a = *static_cast(ap); + int b = *static_cast(bp); + return a < b ? -1 : a > b ? 1 : 0; +} + +// Test stack frames... +void bar() __attribute__((noinline)); + +void bar() { + int a[2] = {1, 2}; + // Use qsort, which is in a different library + qsort(a, 2, sizeof(int), comparator); +} + +class ElfCacheTest : public testing::Test { + protected: + void SetUp(); +}; + +// Capture "golden" stack trace with default-configured Symbolizer +void ElfCacheTest::SetUp() { + bar(); + Symbolizer symbolizer; + symbolizer.symbolize(goldenFrames); + // At least 3 stack frames from us + getStackTrace() + ASSERT_LE(4, goldenFrames.frameCount); +} + +void runElfCacheTest(Symbolizer& symbolizer) { + FrameArray<100> frames = goldenFrames; + for (size_t i = 0; i < frames.frameCount; ++i) { + auto& f = frames.frames[i]; + f.found = false; + f.name.clear(); + } + symbolizer.symbolize(frames); + ASSERT_LE(4, frames.frameCount); + for (size_t i = 1; i < 4; ++i) { + EXPECT_EQ(goldenFrames.frames[i].name, frames.frames[i].name); + } +} + +TEST_F(ElfCacheTest, TinyElfCache) { + ElfCache cache(1); + Symbolizer symbolizer(&cache); + // Run twice, in case the wrong stuff gets evicted? + for (size_t i = 0; i < 2; ++i) { + runElfCacheTest(symbolizer); + } +} + +TEST_F(ElfCacheTest, SignalSafeElfCache) { + SignalSafeElfCache cache(100); + Symbolizer symbolizer(&cache); + for (size_t i = 0; i < 2; ++i) { + runElfCacheTest(symbolizer); + } +} + }}} // namespaces