2 * Copyright 2016 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #ifndef __STDC_FORMAT_MACROS
18 #define __STDC_FORMAT_MACROS
21 #include <folly/io/async/EventBase.h>
23 #include <folly/ThreadName.h>
24 #include <folly/io/async/NotificationQueue.h>
25 #include <folly/portability/Unistd.h>
27 #include <condition_variable>
34 using folly::EventBase;
36 class FunctionLoopCallback : public EventBase::LoopCallback {
38 explicit FunctionLoopCallback(EventBase::Func&& function)
39 : function_(std::move(function)) {}
41 void runLoopCallback() noexcept override {
47 EventBase::Func function_;
54 * EventBase::FunctionRunner
57 class EventBase::FunctionRunner
58 : public NotificationQueue<EventBase::Func>::Consumer {
60 void messageAvailable(Func&& msg) override {
61 // In libevent2, internal events do not break the loop.
62 // Most users would expect loop(), followed by runInEventBaseThread(),
63 // to break the loop and check if it should exit or not.
64 // To have similar bejaviour to libevent1.4, tell the loop to break here.
65 // Note that loop() may still continue to loop, but it will also check the
66 // stop_ flag as well as runInLoop callbacks, etc.
67 event_base_loopbreak(getEventBase()->evb_);
70 // terminateLoopSoon() sends a null message just to
71 // wake up the loop. We can ignore these messages.
75 // The function should never throw an exception, because we have no
76 // way of knowing what sort of error handling to perform.
78 // If it does throw, log a message and abort the program.
81 } catch (const std::exception& ex) {
82 LOG(ERROR) << "runInEventBaseThread() function threw a "
83 << typeid(ex).name() << " exception: " << ex.what();
86 LOG(ERROR) << "runInEventBaseThread() function threw an exception";
92 // The interface used to libevent is not thread-safe. Calls to
93 // event_init() and event_base_free() directly modify an internal
94 // global 'current_base', so a mutex is required to protect this.
96 // event_init() should only ever be called once. Subsequent calls
97 // should be made to event_base_new(). We can recognise that
98 // event_init() has already been called by simply inspecting current_base.
99 static std::mutex libevent_mutex_;
105 EventBase::EventBase(bool enableTimeMeasurement)
106 : runOnceCallbacks_(nullptr)
112 , avgLoopTime_(2000000)
113 , maxLatencyLoopTime_(avgLoopTime_)
114 , enableTimeMeasurement_(enableTimeMeasurement)
115 , nextLoopCnt_(uint64_t(-40)) // Early wrap-around so bugs will manifest soon
116 , latestLoopCnt_(nextLoopCnt_)
119 , observerSampleCount_(0)
120 , executionObserver_(nullptr) {
123 std::lock_guard<std::mutex> lock(libevent_mutex_);
125 // The value 'current_base' (libevent 1) or
126 // 'event_global_current_base_' (libevent 2) is filled in by event_set(),
127 // allowing examination of its value without an explicit reference here.
128 // If ev.ev_base is NULL, then event_init() must be called, otherwise
129 // call event_base_new().
130 event_set(&ev, 0, 0, nullptr, nullptr);
137 evb_ = event_base_new();
140 if (UNLIKELY(evb_ == nullptr)) {
141 LOG(ERROR) << "EventBase(): Failed to init event base.";
142 folly::throwSystemError("error in EventBase::EventBase()");
144 VLOG(5) << "EventBase(): Created.";
145 initNotificationQueue();
146 RequestContext::saveContext();
149 // takes ownership of the event_base
150 EventBase::EventBase(event_base* evb, bool enableTimeMeasurement)
151 : runOnceCallbacks_(nullptr)
158 , avgLoopTime_(2000000)
159 , maxLatencyLoopTime_(avgLoopTime_)
160 , enableTimeMeasurement_(enableTimeMeasurement)
161 , nextLoopCnt_(uint64_t(-40)) // Early wrap-around so bugs will manifest soon
162 , latestLoopCnt_(nextLoopCnt_)
165 , observerSampleCount_(0)
166 , executionObserver_(nullptr) {
167 if (UNLIKELY(evb_ == nullptr)) {
168 LOG(ERROR) << "EventBase(): Pass nullptr as event base.";
169 throw std::invalid_argument("EventBase(): event base cannot be nullptr");
171 initNotificationQueue();
172 RequestContext::saveContext();
175 EventBase::~EventBase() {
176 // Keep looping until all keep-alive handles are released. Each keep-alive
177 // handle signals that some external code will still schedule some work on
178 // this EventBase (so it's not safe to destroy it).
179 while (loopKeepAliveCount() > 0) {
180 applyLoopKeepAlive();
184 // Call all destruction callbacks, before we start cleaning up our state.
185 while (!onDestructionCallbacks_.empty()) {
186 LoopCallback* callback = &onDestructionCallbacks_.front();
187 onDestructionCallbacks_.pop_front();
188 callback->runLoopCallback();
193 while (!runBeforeLoopCallbacks_.empty()) {
194 delete &runBeforeLoopCallbacks_.front();
197 (void)runLoopCallbacks();
199 if (!fnRunner_->consumeUntilDrained()) {
200 LOG(ERROR) << "~EventBase(): Unable to drain notification queue";
203 // Stop consumer before deleting NotificationQueue
204 fnRunner_->stopConsuming();
206 std::lock_guard<std::mutex> lock(libevent_mutex_);
207 event_base_free(evb_);
211 std::lock_guard<std::mutex> lock(localStorageMutex_);
212 for (auto storage : localStorageToDtor_) {
213 storage->onEventBaseDestruction(*this);
216 VLOG(5) << "EventBase(): Destroyed.";
219 size_t EventBase::getNotificationQueueSize() const {
220 return queue_->size();
223 void EventBase::setMaxReadAtOnce(uint32_t maxAtOnce) {
224 fnRunner_->setMaxReadAtOnce(maxAtOnce);
227 // Set smoothing coefficient for loop load average; input is # of milliseconds
228 // for exp(-1) decay.
229 void EventBase::setLoadAvgMsec(uint32_t ms) {
230 assert(enableTimeMeasurement_);
231 uint64_t us = 1000 * ms;
233 maxLatencyLoopTime_.setTimeInterval(us);
234 avgLoopTime_.setTimeInterval(us);
236 LOG(ERROR) << "non-positive arg to setLoadAvgMsec()";
240 void EventBase::resetLoadAvg(double value) {
241 assert(enableTimeMeasurement_);
242 avgLoopTime_.reset(value);
243 maxLatencyLoopTime_.reset(value);
246 static std::chrono::milliseconds
247 getTimeDelta(std::chrono::steady_clock::time_point* prev) {
248 auto result = std::chrono::steady_clock::now() - *prev;
249 *prev = std::chrono::steady_clock::now();
251 return std::chrono::duration_cast<std::chrono::milliseconds>(result);
254 void EventBase::waitUntilRunning() {
255 while (!isRunning()) {
260 // enters the event_base loop -- will only exit when forced to
261 bool EventBase::loop() {
265 bool EventBase::loopOnce(int flags) {
266 return loopBody(flags | EVLOOP_ONCE);
269 bool EventBase::loopBody(int flags) {
270 VLOG(5) << "EventBase(): Starting loop.";
272 DCHECK(!invokingLoop_)
273 << "Your code just tried to loop over an event base from inside another "
274 << "event base loop. Since libevent is not reentrant, this leads to "
275 << "undefined behavior in opt builds. Please fix immediately. For the "
276 << "common case of an inner function that needs to do some synchronous "
277 << "computation on an event-base, replace getEventBase() by a new, "
278 << "stack-allocated EvenBase.";
279 invokingLoop_ = true;
281 invokingLoop_ = false;
285 bool ranLoopCallbacks;
286 bool blocking = !(flags & EVLOOP_NONBLOCK);
287 bool once = (flags & EVLOOP_ONCE);
289 // time-measurement variables.
290 std::chrono::steady_clock::time_point prev;
291 int64_t idleStart = 0;
295 loopThread_.store(pthread_self(), std::memory_order_release);
297 if (!name_.empty()) {
298 setThreadName(name_);
301 if (enableTimeMeasurement_) {
302 prev = std::chrono::steady_clock::now();
303 idleStart = std::chrono::duration_cast<std::chrono::microseconds>(
304 std::chrono::steady_clock::now().time_since_epoch()).count();
307 while (!stop_.load(std::memory_order_acquire)) {
308 applyLoopKeepAlive();
311 // Run the before loop callbacks
312 LoopCallbackList callbacks;
313 callbacks.swap(runBeforeLoopCallbacks_);
315 while(!callbacks.empty()) {
316 auto* item = &callbacks.front();
317 callbacks.pop_front();
318 item->runLoopCallback();
321 // nobody can add loop callbacks from within this thread if
322 // we don't have to handle anything to start with...
323 if (blocking && loopCallbacks_.empty()) {
324 res = event_base_loop(evb_, EVLOOP_ONCE);
326 res = event_base_loop(evb_, EVLOOP_ONCE | EVLOOP_NONBLOCK);
329 ranLoopCallbacks = runLoopCallbacks();
331 if (enableTimeMeasurement_) {
332 busy = std::chrono::duration_cast<std::chrono::microseconds>(
333 std::chrono::steady_clock::now().time_since_epoch()).count() -
335 idle = startWork_ - idleStart;
337 avgLoopTime_.addSample(idle, busy);
338 maxLatencyLoopTime_.addSample(idle, busy);
341 if (observerSampleCount_++ == observer_->getSampleRate()) {
342 observerSampleCount_ = 0;
343 observer_->loopSample(busy, idle);
347 VLOG(11) << "EventBase " << this << " did not timeout "
348 " loop time guess: " << busy + idle <<
349 " idle time: " << idle <<
350 " busy time: " << busy <<
351 " avgLoopTime: " << avgLoopTime_.get() <<
352 " maxLatencyLoopTime: " << maxLatencyLoopTime_.get() <<
353 " maxLatency_: " << maxLatency_ <<
354 " notificationQueueSize: " << getNotificationQueueSize() <<
355 " nothingHandledYet(): "<< nothingHandledYet();
357 // see if our average loop time has exceeded our limit
358 if ((maxLatency_ > 0) &&
359 (maxLatencyLoopTime_.get() > double(maxLatency_))) {
361 // back off temporarily -- don't keep spamming maxLatencyCob_
362 // if we're only a bit over the limit
363 maxLatencyLoopTime_.dampen(0.9);
366 // Our loop run did real work; reset the idle timer
367 idleStart = std::chrono::duration_cast<std::chrono::microseconds>(
368 std::chrono::steady_clock::now().time_since_epoch()).count();
370 VLOG(11) << "EventBase " << this << " did not timeout";
373 // If the event loop indicate that there were no more events, and
374 // we also didn't have any loop callbacks to run, there is nothing left to
376 if (res != 0 && !ranLoopCallbacks) {
377 // Since Notification Queue is marked 'internal' some events may not have
378 // run. Run them manually if so, and continue looping.
380 if (getNotificationQueueSize() > 0) {
381 fnRunner_->handlerReady(0);
387 if (enableTimeMeasurement_) {
388 VLOG(5) << "EventBase " << this << " loop time: " <<
389 getTimeDelta(&prev).count();
396 // Reset stop_ so loop() can be called again
400 LOG(ERROR) << "EventBase: -- error in event loop, res = " << res;
402 } else if (res == 1) {
403 VLOG(5) << "EventBase: ran out of events (exiting loop)!";
404 } else if (res > 1) {
405 LOG(ERROR) << "EventBase: unknown event loop result = " << res;
409 loopThread_.store({}, std::memory_order_release);
411 VLOG(5) << "EventBase(): Done with loop.";
415 ssize_t EventBase::loopKeepAliveCount() {
416 if (loopKeepAliveCountAtomic_.load(std::memory_order_relaxed)) {
417 loopKeepAliveCount_ +=
418 loopKeepAliveCountAtomic_.exchange(0, std::memory_order_relaxed);
420 DCHECK_GE(loopKeepAliveCount_, 0);
421 return loopKeepAliveCount_;
424 void EventBase::applyLoopKeepAlive() {
425 if (loopKeepAliveActive_ && loopKeepAliveCount() == 0) {
426 // Restore the notification queue internal flag
427 fnRunner_->stopConsuming();
428 fnRunner_->startConsumingInternal(this, queue_.get());
429 loopKeepAliveActive_ = false;
430 } else if (!loopKeepAliveActive_ && loopKeepAliveCount() > 0) {
431 // Update the notification queue event to treat it as a normal
432 // (non-internal) event. The notification queue event always remains
433 // installed, and the main loop won't exit with it installed.
434 fnRunner_->stopConsuming();
435 fnRunner_->startConsuming(this, queue_.get());
436 loopKeepAliveActive_ = true;
440 void EventBase::loopForever() {
444 applyLoopKeepAlive();
446 // Make sure notification queue events are treated as normal events.
447 // We can't use loopKeepAlive() here since LoopKeepAlive token can only be
448 // released inside a loop.
449 ++loopKeepAliveCount_;
451 --loopKeepAliveCount_;
457 folly::throwSystemError("error in EventBase::loopForever()");
461 void EventBase::bumpHandlingTime() {
462 if (!enableTimeMeasurement_) {
466 VLOG(11) << "EventBase " << this << " " << __PRETTY_FUNCTION__ <<
467 " (loop) latest " << latestLoopCnt_ << " next " << nextLoopCnt_;
468 if (nothingHandledYet()) {
469 latestLoopCnt_ = nextLoopCnt_;
471 startWork_ = std::chrono::duration_cast<std::chrono::microseconds>(
472 std::chrono::steady_clock::now().time_since_epoch())
475 VLOG(11) << "EventBase " << this << " " << __PRETTY_FUNCTION__
476 << " (loop) startWork_ " << startWork_;
480 void EventBase::terminateLoopSoon() {
481 VLOG(5) << "EventBase(): Received terminateLoopSoon() command.";
483 // Set stop to true, so the event loop will know to exit.
484 // TODO: We should really use an atomic operation here with a release
488 // Call event_base_loopbreak() so that libevent will exit the next time
490 event_base_loopbreak(evb_);
492 // If terminateLoopSoon() is called from another thread,
493 // the EventBase thread might be stuck waiting for events.
494 // In this case, it won't wake up and notice that stop_ is set until it
495 // receives another event. Send an empty frame to the notification queue
496 // so that the event loop will wake up even if there are no other events.
498 // We don't care about the return value of trySendFrame(). If it fails
499 // this likely means the EventBase already has lots of events waiting
502 queue_->putMessage(nullptr);
504 // We don't care if putMessage() fails. This likely means
505 // the EventBase already has lots of events waiting anyway.
509 void EventBase::runInLoop(LoopCallback* callback, bool thisIteration) {
510 DCHECK(isInEventBaseThread());
511 callback->cancelLoopCallback();
512 callback->context_ = RequestContext::saveContext();
513 if (runOnceCallbacks_ != nullptr && thisIteration) {
514 runOnceCallbacks_->push_back(*callback);
516 loopCallbacks_.push_back(*callback);
520 void EventBase::runInLoop(Func cob, bool thisIteration) {
521 DCHECK(isInEventBaseThread());
522 auto wrapper = new FunctionLoopCallback(std::move(cob));
523 wrapper->context_ = RequestContext::saveContext();
524 if (runOnceCallbacks_ != nullptr && thisIteration) {
525 runOnceCallbacks_->push_back(*wrapper);
527 loopCallbacks_.push_back(*wrapper);
531 void EventBase::runOnDestruction(LoopCallback* callback) {
532 std::lock_guard<std::mutex> lg(onDestructionCallbacksMutex_);
533 callback->cancelLoopCallback();
534 onDestructionCallbacks_.push_back(*callback);
537 void EventBase::runBeforeLoop(LoopCallback* callback) {
538 DCHECK(isInEventBaseThread());
539 callback->cancelLoopCallback();
540 runBeforeLoopCallbacks_.push_back(*callback);
543 bool EventBase::runInEventBaseThread(Func fn) {
545 // It will be received by the FunctionRunner in the EventBase's thread.
547 // We try not to schedule nullptr callbacks
549 LOG(ERROR) << "EventBase " << this
550 << ": Scheduling nullptr callbacks is not allowed";
554 // Short-circuit if we are already in our event base
555 if (inRunningEventBaseThread()) {
556 runInLoop(std::move(fn));
562 queue_->putMessage(std::move(fn));
563 } catch (const std::exception& ex) {
564 LOG(ERROR) << "EventBase " << this << ": failed to schedule function "
565 << "for EventBase thread: " << ex.what();
572 bool EventBase::runInEventBaseThreadAndWait(Func fn) {
573 if (inRunningEventBaseThread()) {
574 LOG(ERROR) << "EventBase " << this << ": Waiting in the event loop is not "
581 std::condition_variable cv;
582 runInEventBaseThread([&] {
584 std::unique_lock<std::mutex> l(m);
587 // We cannot release the lock before notify_one, because a spurious
588 // wakeup in the waiting thread may lead to cv and m going out of scope
593 std::unique_lock<std::mutex> l(m);
594 cv.wait(l, [&] { return ready; });
599 bool EventBase::runImmediatelyOrRunInEventBaseThreadAndWait(Func fn) {
600 if (isInEventBaseThread()) {
604 return runInEventBaseThreadAndWait(std::move(fn));
608 bool EventBase::runLoopCallbacks() {
609 if (!loopCallbacks_.empty()) {
611 // Swap the loopCallbacks_ list with a temporary list on our stack.
612 // This way we will only run callbacks scheduled at the time
613 // runLoopCallbacks() was invoked.
615 // If any of these callbacks in turn call runInLoop() to schedule more
616 // callbacks, those new callbacks won't be run until the next iteration
617 // around the event loop. This prevents runInLoop() callbacks from being
618 // able to start file descriptor and timeout based events.
619 LoopCallbackList currentCallbacks;
620 currentCallbacks.swap(loopCallbacks_);
621 runOnceCallbacks_ = ¤tCallbacks;
623 while (!currentCallbacks.empty()) {
624 LoopCallback* callback = ¤tCallbacks.front();
625 currentCallbacks.pop_front();
626 folly::RequestContextScopeGuard rctx(callback->context_);
627 callback->runLoopCallback();
630 runOnceCallbacks_ = nullptr;
636 void EventBase::initNotificationQueue() {
637 // Infinite size queue
638 queue_.reset(new NotificationQueue<Func>());
640 // We allocate fnRunner_ separately, rather than declaring it directly
641 // as a member of EventBase solely so that we don't need to include
642 // NotificationQueue.h from EventBase.h
643 fnRunner_.reset(new FunctionRunner());
645 // Mark this as an internal event, so event_base_loop() will return if
646 // there are no other events besides this one installed.
648 // Most callers don't care about the internal notification queue used by
649 // EventBase. The queue is always installed, so if we did count the queue as
650 // an active event, loop() would never exit with no more events to process.
651 // Users can use loopForever() if they do care about the notification queue.
652 // (This is useful for EventBase threads that do nothing but process
653 // runInEventBaseThread() notifications.)
654 fnRunner_->startConsumingInternal(this, queue_.get());
657 void EventBase::SmoothLoopTime::setTimeInterval(uint64_t timeInterval) {
658 expCoeff_ = -1.0/timeInterval;
659 VLOG(11) << "expCoeff_ " << expCoeff_ << " " << __PRETTY_FUNCTION__;
662 void EventBase::SmoothLoopTime::reset(double value) {
666 void EventBase::SmoothLoopTime::addSample(int64_t idle, int64_t busy) {
668 * Position at which the busy sample is considered to be taken.
669 * (Allows to quickly skew our average without editing much code)
671 enum BusySamplePosition {
672 RIGHT = 0, // busy sample placed at the end of the iteration
673 CENTER = 1, // busy sample placed at the middle point of the iteration
674 LEFT = 2, // busy sample placed at the beginning of the iteration
677 // See http://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
678 // and D676020 for more info on this calculation.
679 VLOG(11) << "idle " << idle << " oldBusyLeftover_ " << oldBusyLeftover_ <<
680 " idle + oldBusyLeftover_ " << idle + oldBusyLeftover_ <<
681 " busy " << busy << " " << __PRETTY_FUNCTION__;
682 idle += oldBusyLeftover_ + busy;
683 oldBusyLeftover_ = (busy * BusySamplePosition::CENTER) / 2;
684 idle -= oldBusyLeftover_;
686 double coeff = exp(idle * expCoeff_);
688 value_ += (1.0 - coeff) * busy;
691 bool EventBase::nothingHandledYet() const noexcept {
692 VLOG(11) << "latest " << latestLoopCnt_ << " next " << nextLoopCnt_;
693 return (nextLoopCnt_ != latestLoopCnt_);
696 void EventBase::attachTimeoutManager(AsyncTimeout* obj,
697 InternalEnum internal) {
699 struct event* ev = obj->getEvent();
700 assert(ev->ev_base == nullptr);
702 event_base_set(getLibeventBase(), ev);
703 if (internal == AsyncTimeout::InternalEnum::INTERNAL) {
704 // Set the EVLIST_INTERNAL flag
705 event_ref_flags(ev) |= EVLIST_INTERNAL;
709 void EventBase::detachTimeoutManager(AsyncTimeout* obj) {
711 struct event* ev = obj->getEvent();
712 ev->ev_base = nullptr;
715 bool EventBase::scheduleTimeout(AsyncTimeout* obj,
716 TimeoutManager::timeout_type timeout) {
717 assert(isInEventBaseThread());
718 // Set up the timeval and add the event
720 tv.tv_sec = long(timeout.count() / 1000LL);
721 tv.tv_usec = long((timeout.count() % 1000LL) * 1000LL);
723 struct event* ev = obj->getEvent();
724 if (event_add(ev, &tv) < 0) {
725 LOG(ERROR) << "EventBase: failed to schedule timeout: " << strerror(errno);
732 void EventBase::cancelTimeout(AsyncTimeout* obj) {
733 assert(isInEventBaseThread());
734 struct event* ev = obj->getEvent();
735 if (EventUtil::isEventRegistered(ev)) {
740 void EventBase::setName(const std::string& name) {
741 assert(isInEventBaseThread());
745 setThreadName(loopThread_.load(std::memory_order_relaxed),
750 const std::string& EventBase::getName() {
751 assert(isInEventBaseThread());
755 const char* EventBase::getLibeventVersion() { return event_get_version(); }
756 const char* EventBase::getLibeventMethod() { return event_get_method(); }