2 * Copyright 2017 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #ifndef __STDC_FORMAT_MACROS
18 #define __STDC_FORMAT_MACROS
21 #include <folly/io/async/EventBase.h>
29 #include <folly/Baton.h>
30 #include <folly/Memory.h>
31 #include <folly/io/async/NotificationQueue.h>
32 #include <folly/io/async/VirtualEventBase.h>
33 #include <folly/portability/Unistd.h>
34 #include <folly/system/ThreadName.h>
39 * EventBase::FunctionRunner
42 class EventBase::FunctionRunner
43 : public NotificationQueue<EventBase::Func>::Consumer {
45 void messageAvailable(Func&& msg) noexcept override {
46 // In libevent2, internal events do not break the loop.
47 // Most users would expect loop(), followed by runInEventBaseThread(),
48 // to break the loop and check if it should exit or not.
49 // To have similar bejaviour to libevent1.4, tell the loop to break here.
50 // Note that loop() may still continue to loop, but it will also check the
51 // stop_ flag as well as runInLoop callbacks, etc.
52 event_base_loopbreak(getEventBase()->evb_);
55 // terminateLoopSoon() sends a null message just to
56 // wake up the loop. We can ignore these messages.
63 // The interface used to libevent is not thread-safe. Calls to
64 // event_init() and event_base_free() directly modify an internal
65 // global 'current_base', so a mutex is required to protect this.
67 // event_init() should only ever be called once. Subsequent calls
68 // should be made to event_base_new(). We can recognise that
69 // event_init() has already been called by simply inspecting current_base.
70 static std::mutex libevent_mutex_;
76 EventBase::EventBase(bool enableTimeMeasurement)
77 : runOnceCallbacks_(nullptr)
83 , avgLoopTime_(std::chrono::seconds(2))
84 , maxLatencyLoopTime_(avgLoopTime_)
85 , enableTimeMeasurement_(enableTimeMeasurement)
86 , nextLoopCnt_(uint64_t(-40)) // Early wrap-around so bugs will manifest soon
87 , latestLoopCnt_(nextLoopCnt_)
90 , observerSampleCount_(0)
91 , executionObserver_(nullptr) {
94 std::lock_guard<std::mutex> lock(libevent_mutex_);
96 // The value 'current_base' (libevent 1) or
97 // 'event_global_current_base_' (libevent 2) is filled in by event_set(),
98 // allowing examination of its value without an explicit reference here.
99 // If ev.ev_base is nullptr, then event_init() must be called, otherwise
100 // call event_base_new().
101 event_set(&ev, 0, 0, nullptr, nullptr);
108 evb_ = event_base_new();
111 if (UNLIKELY(evb_ == nullptr)) {
112 LOG(ERROR) << "EventBase(): Failed to init event base.";
113 folly::throwSystemError("error in EventBase::EventBase()");
115 VLOG(5) << "EventBase(): Created.";
116 initNotificationQueue();
117 RequestContext::saveContext();
120 // takes ownership of the event_base
121 EventBase::EventBase(event_base* evb, bool enableTimeMeasurement)
122 : runOnceCallbacks_(nullptr)
129 , avgLoopTime_(std::chrono::seconds(2))
130 , maxLatencyLoopTime_(avgLoopTime_)
131 , enableTimeMeasurement_(enableTimeMeasurement)
132 , nextLoopCnt_(uint64_t(-40)) // Early wrap-around so bugs will manifest soon
133 , latestLoopCnt_(nextLoopCnt_)
136 , observerSampleCount_(0)
137 , executionObserver_(nullptr) {
138 if (UNLIKELY(evb_ == nullptr)) {
139 LOG(ERROR) << "EventBase(): Pass nullptr as event base.";
140 throw std::invalid_argument("EventBase(): event base cannot be nullptr");
142 initNotificationQueue();
143 RequestContext::saveContext();
146 EventBase::~EventBase() {
147 std::future<void> virtualEventBaseDestroyFuture;
148 if (virtualEventBase_) {
149 virtualEventBaseDestroyFuture = virtualEventBase_->destroy();
152 // Keep looping until all keep-alive handles are released. Each keep-alive
153 // handle signals that some external code will still schedule some work on
154 // this EventBase (so it's not safe to destroy it).
155 while (loopKeepAliveCount() > 0) {
156 applyLoopKeepAlive();
160 if (virtualEventBaseDestroyFuture.valid()) {
161 virtualEventBaseDestroyFuture.get();
164 // Call all destruction callbacks, before we start cleaning up our state.
165 while (!onDestructionCallbacks_.empty()) {
166 LoopCallback* callback = &onDestructionCallbacks_.front();
167 onDestructionCallbacks_.pop_front();
168 callback->runLoopCallback();
173 DCHECK_EQ(0u, runBeforeLoopCallbacks_.size());
175 (void)runLoopCallbacks();
177 if (!fnRunner_->consumeUntilDrained()) {
178 LOG(ERROR) << "~EventBase(): Unable to drain notification queue";
181 // Stop consumer before deleting NotificationQueue
182 fnRunner_->stopConsuming();
184 std::lock_guard<std::mutex> lock(libevent_mutex_);
185 event_base_free(evb_);
188 for (auto storage : localStorageToDtor_) {
189 storage->onEventBaseDestruction(*this);
192 VLOG(5) << "EventBase(): Destroyed.";
195 size_t EventBase::getNotificationQueueSize() const {
196 return queue_->size();
199 void EventBase::setMaxReadAtOnce(uint32_t maxAtOnce) {
200 fnRunner_->setMaxReadAtOnce(maxAtOnce);
203 void EventBase::checkIsInEventBaseThread() const {
204 auto evbTid = loopThread_.load(std::memory_order_relaxed);
205 if (evbTid == std::thread::id()) {
209 // Using getThreadName(evbTid) instead of name_ will work also if
210 // the thread name is set outside of EventBase (and name_ is empty).
211 auto curTid = std::this_thread::get_id();
212 CHECK(evbTid == curTid)
213 << "This logic must be executed in the event base thread. "
214 << "Event base thread name: \""
215 << folly::getThreadName(evbTid).value_or("")
216 << "\", current thread name: \""
217 << folly::getThreadName(curTid).value_or("") << "\"";
220 // Set smoothing coefficient for loop load average; input is # of milliseconds
221 // for exp(-1) decay.
222 void EventBase::setLoadAvgMsec(std::chrono::milliseconds ms) {
223 assert(enableTimeMeasurement_);
224 std::chrono::microseconds us = std::chrono::milliseconds(ms);
225 if (ms > std::chrono::milliseconds::zero()) {
226 maxLatencyLoopTime_.setTimeInterval(us);
227 avgLoopTime_.setTimeInterval(us);
229 LOG(ERROR) << "non-positive arg to setLoadAvgMsec()";
233 void EventBase::resetLoadAvg(double value) {
234 assert(enableTimeMeasurement_);
235 avgLoopTime_.reset(value);
236 maxLatencyLoopTime_.reset(value);
239 static std::chrono::milliseconds
240 getTimeDelta(std::chrono::steady_clock::time_point* prev) {
241 auto result = std::chrono::steady_clock::now() - *prev;
242 *prev = std::chrono::steady_clock::now();
244 return std::chrono::duration_cast<std::chrono::milliseconds>(result);
247 void EventBase::waitUntilRunning() {
248 while (!isRunning()) {
249 std::this_thread::yield();
253 // enters the event_base loop -- will only exit when forced to
254 bool EventBase::loop() {
258 bool EventBase::loopOnce(int flags) {
259 return loopBody(flags | EVLOOP_ONCE);
262 bool EventBase::loopBody(int flags) {
263 VLOG(5) << "EventBase(): Starting loop.";
265 DCHECK(!invokingLoop_)
266 << "Your code just tried to loop over an event base from inside another "
267 << "event base loop. Since libevent is not reentrant, this leads to "
268 << "undefined behavior in opt builds. Please fix immediately. For the "
269 << "common case of an inner function that needs to do some synchronous "
270 << "computation on an event-base, replace getEventBase() by a new, "
271 << "stack-allocated EvenBase.";
272 invokingLoop_ = true;
274 invokingLoop_ = false;
278 bool ranLoopCallbacks;
279 bool blocking = !(flags & EVLOOP_NONBLOCK);
280 bool once = (flags & EVLOOP_ONCE);
282 // time-measurement variables.
283 std::chrono::steady_clock::time_point prev;
284 std::chrono::steady_clock::time_point idleStart = {};
285 std::chrono::microseconds busy;
286 std::chrono::microseconds idle;
288 loopThread_.store(std::this_thread::get_id(), std::memory_order_release);
290 if (!name_.empty()) {
291 setThreadName(name_);
294 if (enableTimeMeasurement_) {
295 prev = std::chrono::steady_clock::now();
296 idleStart = std::chrono::steady_clock::now();
299 while (!stop_.load(std::memory_order_acquire)) {
300 applyLoopKeepAlive();
303 // Run the before loop callbacks
304 LoopCallbackList callbacks;
305 callbacks.swap(runBeforeLoopCallbacks_);
307 while(!callbacks.empty()) {
308 auto* item = &callbacks.front();
309 callbacks.pop_front();
310 item->runLoopCallback();
313 // nobody can add loop callbacks from within this thread if
314 // we don't have to handle anything to start with...
315 if (blocking && loopCallbacks_.empty()) {
316 res = event_base_loop(evb_, EVLOOP_ONCE);
318 res = event_base_loop(evb_, EVLOOP_ONCE | EVLOOP_NONBLOCK);
321 ranLoopCallbacks = runLoopCallbacks();
323 if (enableTimeMeasurement_) {
324 busy = std::chrono::duration_cast<std::chrono::microseconds>(
325 std::chrono::steady_clock::now() - startWork_);
326 idle = std::chrono::duration_cast<std::chrono::microseconds>(
327 startWork_ - idleStart);
329 avgLoopTime_.addSample(std::chrono::microseconds(idle),
330 std::chrono::microseconds(busy));
331 maxLatencyLoopTime_.addSample(std::chrono::microseconds(idle),
332 std::chrono::microseconds(busy));
335 if (observerSampleCount_++ == observer_->getSampleRate()) {
336 observerSampleCount_ = 0;
337 observer_->loopSample(busy.count(), idle.count());
341 VLOG(11) << "EventBase " << this << " did not timeout " <<
342 " loop time guess: " << (busy + idle).count() <<
343 " idle time: " << idle.count() <<
344 " busy time: " << busy.count() <<
345 " avgLoopTime: " << avgLoopTime_.get() <<
346 " maxLatencyLoopTime: " << maxLatencyLoopTime_.get() <<
347 " maxLatency_: " << maxLatency_.count() << "us" <<
348 " notificationQueueSize: " << getNotificationQueueSize() <<
349 " nothingHandledYet(): " << nothingHandledYet();
351 // see if our average loop time has exceeded our limit
352 if ((maxLatency_ > std::chrono::microseconds::zero()) &&
353 (maxLatencyLoopTime_.get() > double(maxLatency_.count()))) {
355 // back off temporarily -- don't keep spamming maxLatencyCob_
356 // if we're only a bit over the limit
357 maxLatencyLoopTime_.dampen(0.9);
360 // Our loop run did real work; reset the idle timer
361 idleStart = std::chrono::steady_clock::now();
363 VLOG(11) << "EventBase " << this << " did not timeout";
366 // If the event loop indicate that there were no more events, and
367 // we also didn't have any loop callbacks to run, there is nothing left to
369 if (res != 0 && !ranLoopCallbacks) {
370 // Since Notification Queue is marked 'internal' some events may not have
371 // run. Run them manually if so, and continue looping.
373 if (getNotificationQueueSize() > 0) {
374 fnRunner_->handlerReady(0);
380 if (enableTimeMeasurement_) {
381 VLOG(11) << "EventBase " << this << " loop time: " <<
382 getTimeDelta(&prev).count();
389 // Reset stop_ so loop() can be called again
393 LOG(ERROR) << "EventBase: -- error in event loop, res = " << res;
395 } else if (res == 1) {
396 VLOG(5) << "EventBase: ran out of events (exiting loop)!";
397 } else if (res > 1) {
398 LOG(ERROR) << "EventBase: unknown event loop result = " << res;
402 loopThread_.store({}, std::memory_order_release);
404 VLOG(5) << "EventBase(): Done with loop.";
408 ssize_t EventBase::loopKeepAliveCount() {
409 if (loopKeepAliveCountAtomic_.load(std::memory_order_relaxed)) {
410 loopKeepAliveCount_ +=
411 loopKeepAliveCountAtomic_.exchange(0, std::memory_order_relaxed);
413 DCHECK_GE(loopKeepAliveCount_, 0);
415 return loopKeepAliveCount_;
418 void EventBase::applyLoopKeepAlive() {
419 auto keepAliveCount = loopKeepAliveCount();
420 // Make sure default VirtualEventBase won't hold EventBase::loop() forever.
421 if (virtualEventBase_ && virtualEventBase_->keepAliveCount() == 1) {
425 if (loopKeepAliveActive_ && keepAliveCount == 0) {
426 // Restore the notification queue internal flag
427 fnRunner_->stopConsuming();
428 fnRunner_->startConsumingInternal(this, queue_.get());
429 loopKeepAliveActive_ = false;
430 } else if (!loopKeepAliveActive_ && keepAliveCount > 0) {
431 // Update the notification queue event to treat it as a normal
432 // (non-internal) event. The notification queue event always remains
433 // installed, and the main loop won't exit with it installed.
434 fnRunner_->stopConsuming();
435 fnRunner_->startConsuming(this, queue_.get());
436 loopKeepAliveActive_ = true;
440 void EventBase::loopForever() {
444 applyLoopKeepAlive();
446 // Make sure notification queue events are treated as normal events.
447 // We can't use loopKeepAlive() here since LoopKeepAlive token can only be
448 // released inside a loop.
449 ++loopKeepAliveCount_;
451 --loopKeepAliveCount_;
457 folly::throwSystemError("error in EventBase::loopForever()");
461 void EventBase::bumpHandlingTime() {
462 if (!enableTimeMeasurement_) {
466 VLOG(11) << "EventBase " << this << " " << __PRETTY_FUNCTION__ <<
467 " (loop) latest " << latestLoopCnt_ << " next " << nextLoopCnt_;
468 if (nothingHandledYet()) {
469 latestLoopCnt_ = nextLoopCnt_;
471 startWork_ = std::chrono::steady_clock::now();
473 VLOG(11) << "EventBase " << this << " " << __PRETTY_FUNCTION__
474 << " (loop) startWork_ " << startWork_.time_since_epoch().count();
478 void EventBase::terminateLoopSoon() {
479 VLOG(5) << "EventBase(): Received terminateLoopSoon() command.";
481 // Set stop to true, so the event loop will know to exit.
482 // TODO: We should really use an atomic operation here with a release
486 // Call event_base_loopbreak() so that libevent will exit the next time
488 event_base_loopbreak(evb_);
490 // If terminateLoopSoon() is called from another thread,
491 // the EventBase thread might be stuck waiting for events.
492 // In this case, it won't wake up and notice that stop_ is set until it
493 // receives another event. Send an empty frame to the notification queue
494 // so that the event loop will wake up even if there are no other events.
496 // We don't care about the return value of trySendFrame(). If it fails
497 // this likely means the EventBase already has lots of events waiting
500 queue_->putMessage(nullptr);
502 // We don't care if putMessage() fails. This likely means
503 // the EventBase already has lots of events waiting anyway.
507 void EventBase::runInLoop(LoopCallback* callback, bool thisIteration) {
508 dcheckIsInEventBaseThread();
509 callback->cancelLoopCallback();
510 callback->context_ = RequestContext::saveContext();
511 if (runOnceCallbacks_ != nullptr && thisIteration) {
512 runOnceCallbacks_->push_back(*callback);
514 loopCallbacks_.push_back(*callback);
518 void EventBase::runInLoop(Func cob, bool thisIteration) {
519 dcheckIsInEventBaseThread();
520 auto wrapper = new FunctionLoopCallback(std::move(cob));
521 wrapper->context_ = RequestContext::saveContext();
522 if (runOnceCallbacks_ != nullptr && thisIteration) {
523 runOnceCallbacks_->push_back(*wrapper);
525 loopCallbacks_.push_back(*wrapper);
529 void EventBase::runOnDestruction(LoopCallback* callback) {
530 std::lock_guard<std::mutex> lg(onDestructionCallbacksMutex_);
531 callback->cancelLoopCallback();
532 onDestructionCallbacks_.push_back(*callback);
535 void EventBase::runBeforeLoop(LoopCallback* callback) {
536 dcheckIsInEventBaseThread();
537 callback->cancelLoopCallback();
538 runBeforeLoopCallbacks_.push_back(*callback);
541 bool EventBase::runInEventBaseThread(Func fn) {
543 // It will be received by the FunctionRunner in the EventBase's thread.
545 // We try not to schedule nullptr callbacks
547 LOG(ERROR) << "EventBase " << this
548 << ": Scheduling nullptr callbacks is not allowed";
552 // Short-circuit if we are already in our event base
553 if (inRunningEventBaseThread()) {
554 runInLoop(std::move(fn));
559 queue_->putMessage(std::move(fn));
560 } catch (const std::exception& ex) {
561 LOG(ERROR) << "EventBase " << this << ": failed to schedule function "
562 << "for EventBase thread: " << ex.what();
569 bool EventBase::runInEventBaseThreadAndWait(Func fn) {
570 if (inRunningEventBaseThread()) {
571 LOG(ERROR) << "EventBase " << this << ": Waiting in the event loop is not "
577 runInEventBaseThread([&ready, fn = std::move(fn)]() mutable {
581 // A trick to force the stored functor to be executed and then destructed
582 // before posting the baton and waking the waiting thread.
583 copy(std::move(fn))();
590 bool EventBase::runImmediatelyOrRunInEventBaseThreadAndWait(Func fn) {
591 if (isInEventBaseThread()) {
595 return runInEventBaseThreadAndWait(std::move(fn));
599 bool EventBase::runLoopCallbacks() {
600 if (!loopCallbacks_.empty()) {
602 // Swap the loopCallbacks_ list with a temporary list on our stack.
603 // This way we will only run callbacks scheduled at the time
604 // runLoopCallbacks() was invoked.
606 // If any of these callbacks in turn call runInLoop() to schedule more
607 // callbacks, those new callbacks won't be run until the next iteration
608 // around the event loop. This prevents runInLoop() callbacks from being
609 // able to start file descriptor and timeout based events.
610 LoopCallbackList currentCallbacks;
611 currentCallbacks.swap(loopCallbacks_);
612 runOnceCallbacks_ = ¤tCallbacks;
614 while (!currentCallbacks.empty()) {
615 LoopCallback* callback = ¤tCallbacks.front();
616 currentCallbacks.pop_front();
617 folly::RequestContextScopeGuard rctx(callback->context_);
618 callback->runLoopCallback();
621 runOnceCallbacks_ = nullptr;
627 void EventBase::initNotificationQueue() {
628 // Infinite size queue
629 queue_ = std::make_unique<NotificationQueue<Func>>();
631 // We allocate fnRunner_ separately, rather than declaring it directly
632 // as a member of EventBase solely so that we don't need to include
633 // NotificationQueue.h from EventBase.h
634 fnRunner_ = std::make_unique<FunctionRunner>();
636 // Mark this as an internal event, so event_base_loop() will return if
637 // there are no other events besides this one installed.
639 // Most callers don't care about the internal notification queue used by
640 // EventBase. The queue is always installed, so if we did count the queue as
641 // an active event, loop() would never exit with no more events to process.
642 // Users can use loopForever() if they do care about the notification queue.
643 // (This is useful for EventBase threads that do nothing but process
644 // runInEventBaseThread() notifications.)
645 fnRunner_->startConsumingInternal(this, queue_.get());
648 void EventBase::SmoothLoopTime::setTimeInterval(
649 std::chrono::microseconds timeInterval) {
650 expCoeff_ = -1.0 / timeInterval.count();
651 VLOG(11) << "expCoeff_ " << expCoeff_ << " " << __PRETTY_FUNCTION__;
654 void EventBase::SmoothLoopTime::reset(double value) {
658 void EventBase::SmoothLoopTime::addSample(
659 std::chrono::microseconds idle,
660 std::chrono::microseconds busy) {
662 * Position at which the busy sample is considered to be taken.
663 * (Allows to quickly skew our average without editing much code)
665 enum BusySamplePosition {
666 RIGHT = 0, // busy sample placed at the end of the iteration
667 CENTER = 1, // busy sample placed at the middle point of the iteration
668 LEFT = 2, // busy sample placed at the beginning of the iteration
671 // See http://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
672 // and D676020 for more info on this calculation.
673 VLOG(11) << "idle " << idle.count() << " oldBusyLeftover_ "
674 << oldBusyLeftover_.count() << " idle + oldBusyLeftover_ "
675 << (idle + oldBusyLeftover_).count() << " busy " << busy.count()
676 << " " << __PRETTY_FUNCTION__;
677 idle += oldBusyLeftover_ + busy;
678 oldBusyLeftover_ = (busy * BusySamplePosition::CENTER) / 2;
679 idle -= oldBusyLeftover_;
681 double coeff = exp(idle.count() * expCoeff_);
683 value_ += (1.0 - coeff) * busy.count();
686 bool EventBase::nothingHandledYet() const noexcept {
687 VLOG(11) << "latest " << latestLoopCnt_ << " next " << nextLoopCnt_;
688 return (nextLoopCnt_ != latestLoopCnt_);
691 void EventBase::attachTimeoutManager(AsyncTimeout* obj,
692 InternalEnum internal) {
694 struct event* ev = obj->getEvent();
695 assert(ev->ev_base == nullptr);
697 event_base_set(getLibeventBase(), ev);
698 if (internal == AsyncTimeout::InternalEnum::INTERNAL) {
699 // Set the EVLIST_INTERNAL flag
700 event_ref_flags(ev) |= EVLIST_INTERNAL;
704 void EventBase::detachTimeoutManager(AsyncTimeout* obj) {
706 struct event* ev = obj->getEvent();
707 ev->ev_base = nullptr;
710 bool EventBase::scheduleTimeout(AsyncTimeout* obj,
711 TimeoutManager::timeout_type timeout) {
712 dcheckIsInEventBaseThread();
713 // Set up the timeval and add the event
715 tv.tv_sec = long(timeout.count() / 1000LL);
716 tv.tv_usec = long((timeout.count() % 1000LL) * 1000LL);
718 struct event* ev = obj->getEvent();
719 if (event_add(ev, &tv) < 0) {
720 LOG(ERROR) << "EventBase: failed to schedule timeout: " << strerror(errno);
727 void EventBase::cancelTimeout(AsyncTimeout* obj) {
728 dcheckIsInEventBaseThread();
729 struct event* ev = obj->getEvent();
730 if (EventUtil::isEventRegistered(ev)) {
735 void EventBase::setName(const std::string& name) {
736 dcheckIsInEventBaseThread();
740 setThreadName(loopThread_.load(std::memory_order_relaxed),
745 const std::string& EventBase::getName() {
746 dcheckIsInEventBaseThread();
750 const char* EventBase::getLibeventVersion() { return event_get_version(); }
751 const char* EventBase::getLibeventMethod() { return event_get_method(); }
753 VirtualEventBase& EventBase::getVirtualEventBase() {
754 folly::call_once(virtualEventBaseInitFlag_, [&] {
755 virtualEventBase_ = std::make_unique<VirtualEventBase>(*this);
758 return *virtualEventBase_;