2 * Copyright 2017 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
16 #include "FiberManagerInternal.h"
23 #include <glog/logging.h>
25 #include <folly/fibers/Fiber.h>
26 #include <folly/fibers/LoopController.h>
28 #include <folly/ConstexprMath.h>
29 #include <folly/SingletonThreadLocal.h>
30 #include <folly/portability/SysSyscall.h>
31 #include <folly/portability/Unistd.h>
33 #ifdef FOLLY_SANITIZE_ADDRESS
37 static void __sanitizer_start_switch_fiber_weak(
38 void** fake_stack_save,
39 void const* fiber_stack_base,
40 size_t fiber_stack_extent)
41 __attribute__((__weakref__("__sanitizer_start_switch_fiber")));
42 static void __sanitizer_finish_switch_fiber_weak(
43 void* fake_stack_save,
44 void const** old_stack_base,
45 size_t* old_stack_extent)
46 __attribute__((__weakref__("__sanitizer_finish_switch_fiber")));
47 static void __asan_unpoison_memory_region_weak(
48 void const /* nolint */ volatile* addr,
49 size_t size) __attribute__((__weakref__("__asan_unpoison_memory_region")));
51 typedef void (*AsanStartSwitchStackFuncPtr)(void**, void const*, size_t);
52 typedef void (*AsanFinishSwitchStackFuncPtr)(void*, void const**, size_t*);
53 typedef void (*AsanUnpoisonMemoryRegionFuncPtr)(
54 void const /* nolint */ volatile*,
60 static AsanStartSwitchStackFuncPtr getStartSwitchStackFunc();
61 static AsanFinishSwitchStackFuncPtr getFinishSwitchStackFunc();
62 static AsanUnpoisonMemoryRegionFuncPtr getUnpoisonMemoryRegionFunc();
71 FOLLY_TLS FiberManager* FiberManager::currentFiberManager_ = nullptr;
73 FiberManager::FiberManager(
74 std::unique_ptr<LoopController> loopController,
78 std::move(loopController),
79 std::move(options)) {}
81 FiberManager::~FiberManager() {
82 if (isLoopScheduled_) {
83 loopController_->cancel();
86 while (!fibersPool_.empty()) {
87 fibersPool_.pop_front_and_dispose([](Fiber* fiber) { delete fiber; });
89 assert(readyFibers_.empty());
90 assert(fibersActive_ == 0);
93 LoopController& FiberManager::loopController() {
94 return *loopController_;
97 const LoopController& FiberManager::loopController() const {
98 return *loopController_;
101 bool FiberManager::hasTasks() const {
102 return fibersActive_ > 0 || !remoteReadyQueue_.empty() ||
103 !remoteTaskQueue_.empty();
106 Fiber* FiberManager::getFiber() {
107 Fiber* fiber = nullptr;
109 if (options_.fibersPoolResizePeriodMs > 0 && !fibersPoolResizerScheduled_) {
110 fibersPoolResizer_();
111 fibersPoolResizerScheduled_ = true;
114 if (fibersPool_.empty()) {
115 fiber = new Fiber(*this);
118 fiber = &fibersPool_.front();
119 fibersPool_.pop_front();
120 assert(fibersPoolSize_ > 0);
124 if (++fibersActive_ > maxFibersActiveLastPeriod_) {
125 maxFibersActiveLastPeriod_ = fibersActive_;
128 bool recordStack = (options_.recordStackEvery != 0) &&
129 (fiberId_ % options_.recordStackEvery == 0);
130 fiber->init(recordStack);
134 void FiberManager::setExceptionCallback(FiberManager::ExceptionCallback ec) {
136 exceptionCallback_ = std::move(ec);
139 size_t FiberManager::fibersAllocated() const {
140 return fibersAllocated_;
143 size_t FiberManager::fibersPoolSize() const {
144 return fibersPoolSize_;
147 size_t FiberManager::stackHighWatermark() const {
148 return stackHighWatermark_;
151 void FiberManager::remoteReadyInsert(Fiber* fiber) {
153 observer_->runnable(reinterpret_cast<uintptr_t>(fiber));
155 auto insertHead = [&]() { return remoteReadyQueue_.insertHead(fiber); };
156 loopController_->scheduleThreadSafe(std::ref(insertHead));
159 void FiberManager::setObserver(ExecutionObserver* observer) {
160 observer_ = observer;
163 ExecutionObserver* FiberManager::getObserver() {
167 void FiberManager::setPreemptRunner(InlineFunctionRunner* preemptRunner) {
168 preemptRunner_ = preemptRunner;
171 void FiberManager::doFibersPoolResizing() {
172 while (fibersAllocated_ > maxFibersActiveLastPeriod_ &&
173 fibersPoolSize_ > options_.maxFibersPoolSize) {
174 auto fiber = &fibersPool_.front();
175 assert(fiber != nullptr);
176 fibersPool_.pop_front();
182 maxFibersActiveLastPeriod_ = fibersActive_;
185 void FiberManager::FibersPoolResizer::operator()() {
186 fiberManager_.doFibersPoolResizing();
187 fiberManager_.timeoutManager_->registerTimeout(
189 std::chrono::milliseconds(
190 fiberManager_.options_.fibersPoolResizePeriodMs));
193 #ifdef FOLLY_SANITIZE_ADDRESS
195 void FiberManager::registerStartSwitchStackWithAsan(
196 void** saveFakeStack,
197 const void* stackBottom,
199 // Check if we can find a fiber enter function and call it if we find one
200 static AsanStartSwitchStackFuncPtr fn = getStartSwitchStackFunc();
202 LOG(FATAL) << "The version of ASAN in use doesn't support fibers";
204 fn(saveFakeStack, stackBottom, stackSize);
208 void FiberManager::registerFinishSwitchStackWithAsan(
210 const void** saveStackBottom,
211 size_t* saveStackSize) {
212 // Check if we can find a fiber exit function and call it if we find one
213 static AsanFinishSwitchStackFuncPtr fn = getFinishSwitchStackFunc();
215 LOG(FATAL) << "The version of ASAN in use doesn't support fibers";
217 fn(saveFakeStack, saveStackBottom, saveStackSize);
221 void FiberManager::freeFakeStack(void* fakeStack) {
222 static AsanStartSwitchStackFuncPtr fnStart = getStartSwitchStackFunc();
223 static AsanFinishSwitchStackFuncPtr fnFinish = getFinishSwitchStackFunc();
224 if (fnStart == nullptr || fnFinish == nullptr) {
225 LOG(FATAL) << "The version of ASAN in use doesn't support fibers";
229 const void* stackBottom;
231 fnStart(&saveFakeStack, nullptr, 0);
232 fnFinish(fakeStack, &stackBottom, &stackSize);
233 fnStart(nullptr, stackBottom, stackSize);
234 fnFinish(saveFakeStack, nullptr, nullptr);
237 void FiberManager::unpoisonFiberStack(const Fiber* fiber) {
238 auto stack = fiber->getStack();
240 // Check if we can find a fiber enter function and call it if we find one
241 static AsanUnpoisonMemoryRegionFuncPtr fn = getUnpoisonMemoryRegionFunc();
243 LOG(FATAL) << "This version of ASAN doesn't support memory unpoisoning";
245 fn(stack.first, stack.second);
249 static AsanStartSwitchStackFuncPtr getStartSwitchStackFunc() {
250 AsanStartSwitchStackFuncPtr fn{nullptr};
252 // Check whether weak reference points to statically linked enter function
253 if (nullptr != (fn = &::__sanitizer_start_switch_fiber_weak)) {
257 // Check whether we can find a dynamically linked enter function
259 (fn = (AsanStartSwitchStackFuncPtr)dlsym(
260 RTLD_DEFAULT, "__sanitizer_start_switch_fiber"))) {
264 // Couldn't find the function at all
268 static AsanFinishSwitchStackFuncPtr getFinishSwitchStackFunc() {
269 AsanFinishSwitchStackFuncPtr fn{nullptr};
271 // Check whether weak reference points to statically linked exit function
272 if (nullptr != (fn = &::__sanitizer_finish_switch_fiber_weak)) {
276 // Check whether we can find a dynamically linked exit function
278 (fn = (AsanFinishSwitchStackFuncPtr)dlsym(
279 RTLD_DEFAULT, "__sanitizer_finish_switch_fiber"))) {
283 // Couldn't find the function at all
287 static AsanUnpoisonMemoryRegionFuncPtr getUnpoisonMemoryRegionFunc() {
288 AsanUnpoisonMemoryRegionFuncPtr fn{nullptr};
290 // Check whether weak reference points to statically linked unpoison function
291 if (nullptr != (fn = &::__asan_unpoison_memory_region_weak)) {
295 // Check whether we can find a dynamically linked unpoison function
297 (fn = (AsanUnpoisonMemoryRegionFuncPtr)dlsym(
298 RTLD_DEFAULT, "__asan_unpoison_memory_region"))) {
302 // Couldn't find the function at all
306 #endif // FOLLY_SANITIZE_ADDRESS
311 // SIGSTKSZ (8 kB on our architectures) isn't always enough for
312 // folly::symbolizer, so allocate 32 kB.
313 constexpr size_t kAltStackSize = folly::constexpr_max(SIGSTKSZ, 32 * 1024);
315 bool hasAlternateStack() {
317 sigaltstack(nullptr, &ss);
318 return !(ss.ss_flags & SS_DISABLE);
321 int setAlternateStack(char* sp, size_t size) {
326 return sigaltstack(&ss, nullptr);
329 int unsetAlternateStack() {
331 ss.ss_flags = SS_DISABLE;
332 return sigaltstack(&ss, nullptr);
335 class ScopedAlternateSignalStack {
337 ScopedAlternateSignalStack() {
338 if (hasAlternateStack()) {
342 stack_ = std::make_unique<AltStackBuffer>();
344 setAlternateStack(stack_->data(), stack_->size());
347 ~ScopedAlternateSignalStack() {
349 unsetAlternateStack();
354 using AltStackBuffer = std::array<char, kAltStackSize>;
355 std::unique_ptr<AltStackBuffer> stack_;
359 void FiberManager::registerAlternateSignalStack() {
360 static folly::SingletonThreadLocal<ScopedAlternateSignalStack> singleton;
363 alternateSignalStackRegistered_ = true;