/*
- * Copyright 2016 Facebook, Inc.
+ * Copyright 2017 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
#include <vector>
#include <folly/Memory.h>
+#include <folly/Random.h>
#include <folly/futures/Future.h>
#include <folly/Conv.h>
#include <folly/fibers/AddTasks.h>
+#include <folly/fibers/AtomicBatchDispatcher.h>
#include <folly/fibers/BatchDispatcher.h>
#include <folly/fibers/EventBaseLoopController.h>
#include <folly/fibers/FiberManager.h>
#include <folly/fibers/GenericBaton.h>
#include <folly/fibers/Semaphore.h>
#include <folly/fibers/SimpleLoopController.h>
+#include <folly/fibers/TimedMutex.h>
#include <folly/fibers/WhenN.h>
+#include <folly/io/async/ScopedEventBaseThread.h>
#include <folly/portability/GTest.h>
using namespace folly::fibers;
if (!taskAdded) {
manager.addTask([&]() {
std::vector<std::function<std::unique_ptr<int>()>> funcs;
- for (size_t i = 0; i < 3; ++i) {
+ for (int i = 0; i < 3; ++i) {
funcs.push_back([i, &pendingFibers]() {
await([&pendingFibers](Promise<int> promise) {
pendingFibers.push_back(std::move(promise));
manager.addTask([&]() {
std::vector<std::function<int()>> funcs;
for (size_t i = 0; i < 3; ++i) {
- funcs.push_back([i, &pendingFibers]() {
+ funcs.push_back([i, &pendingFibers]() -> size_t {
await([&pendingFibers](Promise<int> promise) {
pendingFibers.push_back(std::move(promise));
});
throw std::runtime_error("Runtime");
- return i * 2 + 1;
});
}
for (size_t i = 0; i < kTasks; ++i) {
manager.addTask([&, completionCounter]() {
- for (size_t i = 0; i < kIterations; ++i) {
+ for (size_t j = 0; j < kIterations; ++j) {
sem.wait();
++counter;
sem.signal();
evb.loop();
}
+namespace AtomicBatchDispatcherTesting {
+
+using ValueT = size_t;
+using ResultT = std::string;
+using DispatchFunctionT =
+ folly::Function<std::vector<ResultT>(std::vector<ValueT>&&)>;
+
+#define ENABLE_TRACE_IN_TEST 0 // Set to 1 to debug issues in ABD tests
+#if ENABLE_TRACE_IN_TEST
+#define OUTPUT_TRACE std::cerr
+#else // ENABLE_TRACE_IN_TEST
+struct DevNullPiper {
+ template <typename T>
+ DevNullPiper& operator<<(const T&) {
+ return *this;
+ }
+
+ DevNullPiper& operator<<(std::ostream& (*)(std::ostream&)) {
+ return *this;
+ }
+} devNullPiper;
+#define OUTPUT_TRACE devNullPiper
+#endif // ENABLE_TRACE_IN_TEST
+
+struct Job {
+ AtomicBatchDispatcher<ValueT, ResultT>::Token token;
+ ValueT input;
+
+ void preprocess(FiberManager& executor, bool die) {
+ // Yield for a random duration [0, 10] ms to simulate I/O in preprocessing
+ clock_t msecToDoIO = folly::Random::rand32() % 10;
+ double start = (1000.0 * clock()) / CLOCKS_PER_SEC;
+ double endAfter = start + msecToDoIO;
+ while ((1000.0 * clock()) / CLOCKS_PER_SEC < endAfter) {
+ executor.yield();
+ }
+ if (die) {
+ throw std::logic_error("Simulating preprocessing failure");
+ }
+ }
+
+ Job(AtomicBatchDispatcher<ValueT, ResultT>::Token&& t, ValueT i)
+ : token(std::move(t)), input(i) {}
+
+ Job(Job&&) = default;
+ Job& operator=(Job&&) = default;
+};
+
+ResultT processSingleInput(ValueT&& input) {
+ return folly::to<ResultT>(std::move(input));
+}
+
+std::vector<ResultT> userDispatchFunc(std::vector<ValueT>&& inputs) {
+ size_t expectedCount = inputs.size();
+ std::vector<ResultT> results;
+ results.reserve(expectedCount);
+ for (size_t i = 0; i < expectedCount; ++i) {
+ results.emplace_back(processSingleInput(std::move(inputs[i])));
+ }
+ return results;
+}
+
+void createJobs(
+ AtomicBatchDispatcher<ValueT, ResultT>& atomicBatchDispatcher,
+ std::vector<Job>& jobs,
+ size_t count) {
+ jobs.clear();
+ for (size_t i = 0; i < count; ++i) {
+ jobs.emplace_back(Job(atomicBatchDispatcher.getToken(), i));
+ }
+}
+
+enum class DispatchProblem {
+ None,
+ PreprocessThrows,
+ DuplicateDispatch,
+};
+
+void dispatchJobs(
+ FiberManager& executor,
+ std::vector<Job>& jobs,
+ std::vector<folly::Optional<folly::Future<ResultT>>>& results,
+ DispatchProblem dispatchProblem = DispatchProblem::None,
+ size_t problemIndex = size_t(-1)) {
+ EXPECT_TRUE(
+ dispatchProblem == DispatchProblem::None || problemIndex < jobs.size());
+ results.clear();
+ results.resize(jobs.size());
+ for (size_t i = 0; i < jobs.size(); ++i) {
+ executor.add(
+ [i, &executor, &jobs, &results, dispatchProblem, problemIndex]() {
+ try {
+ Job job(std::move(jobs[i]));
+
+ if (dispatchProblem == DispatchProblem::PreprocessThrows) {
+ if (i == problemIndex) {
+ EXPECT_THROW(job.preprocess(executor, true), std::logic_error);
+ return;
+ }
+ }
+
+ job.preprocess(executor, false);
+ OUTPUT_TRACE << "Dispatching job #" << i << std::endl;
+ results[i] = job.token.dispatch(job.input);
+ OUTPUT_TRACE << "Result future filled for job #" << i << std::endl;
+
+ if (dispatchProblem == DispatchProblem::DuplicateDispatch) {
+ if (i == problemIndex) {
+ EXPECT_THROW(job.token.dispatch(job.input), ABDUsageException);
+ }
+ }
+ } catch (...) {
+ OUTPUT_TRACE << "Preprocessing failed for job #" << i << std::endl;
+ }
+ });
+ }
+}
+
+void validateResult(
+ std::vector<folly::Optional<folly::Future<ResultT>>>& results,
+ size_t i) {
+ try {
+ OUTPUT_TRACE << "results[" << i << "].value() : " << results[i]->value()
+ << std::endl;
+ } catch (std::exception& e) {
+ OUTPUT_TRACE << "Exception : " << e.what() << std::endl;
+ throw;
+ }
+}
+
+template <typename TException>
+void validateResults(
+ std::vector<folly::Optional<folly::Future<ResultT>>>& results,
+ size_t expectedNumResults) {
+ size_t numResultsFilled = 0;
+ for (size_t i = 0; i < results.size(); ++i) {
+ if (!results[i]) {
+ continue;
+ }
+ ++numResultsFilled;
+ EXPECT_THROW(validateResult(results, i), TException);
+ }
+ EXPECT_EQ(numResultsFilled, expectedNumResults);
+}
+
+void validateResults(
+ std::vector<folly::Optional<folly::Future<ResultT>>>& results,
+ size_t expectedNumResults) {
+ size_t numResultsFilled = 0;
+ for (size_t i = 0; i < results.size(); ++i) {
+ if (!results[i]) {
+ continue;
+ }
+ ++numResultsFilled;
+ EXPECT_NO_THROW(validateResult(results, i));
+ ValueT expectedInput = i;
+ EXPECT_EQ(
+ results[i]->value(), processSingleInput(std::move(expectedInput)));
+ }
+ EXPECT_EQ(numResultsFilled, expectedNumResults);
+}
+
+} // AtomicBatchDispatcherTesting
+
+#define SET_UP_TEST_FUNC \
+ using namespace AtomicBatchDispatcherTesting; \
+ folly::EventBase evb; \
+ auto& executor = getFiberManager(evb); \
+ const size_t COUNT = 11; \
+ std::vector<Job> jobs; \
+ jobs.reserve(COUNT); \
+ std::vector<folly::Optional<folly::Future<ResultT>>> results; \
+ results.reserve(COUNT); \
+ DispatchFunctionT dispatchFunc
+
+TEST(FiberManager, ABD_Test) {
+ SET_UP_TEST_FUNC;
+
+ //
+ // Testing AtomicBatchDispatcher with explicit call to commit()
+ //
+ dispatchFunc = userDispatchFunc;
+ auto atomicBatchDispatcher =
+ createAtomicBatchDispatcher(std::move(dispatchFunc));
+ createJobs(atomicBatchDispatcher, jobs, COUNT);
+ dispatchJobs(executor, jobs, results);
+ atomicBatchDispatcher.commit();
+ evb.loop();
+ validateResults(results, COUNT);
+}
+
+TEST(FiberManager, ABD_DispatcherDestroyedBeforeCallingCommit) {
+ SET_UP_TEST_FUNC;
+
+ //
+ // Testing AtomicBatchDispatcher destroyed before calling commit.
+ // Handles error cases for:
+ // - User might have forgotten to add the call to commit() in the code
+ // - An unexpected exception got thrown in user code before commit() is called
+ //
+ try {
+ dispatchFunc = userDispatchFunc;
+ auto atomicBatchDispatcher =
+ createAtomicBatchDispatcher(std::move(dispatchFunc));
+ createJobs(atomicBatchDispatcher, jobs, COUNT);
+ dispatchJobs(executor, jobs, results);
+ throw std::runtime_error(
+ "Unexpected exception in user code before commit called");
+ // atomicBatchDispatcher.commit();
+ } catch (...) {
+ /* User code handles the exception and does not exit process */
+ }
+ evb.loop();
+ validateResults<ABDCommitNotCalledException>(results, COUNT);
+}
+
+TEST(FiberManager, ABD_PreprocessingFailureTest) {
+ SET_UP_TEST_FUNC;
+
+ //
+ // Testing preprocessing failure on a job throws
+ //
+ dispatchFunc = userDispatchFunc;
+ auto atomicBatchDispatcher =
+ createAtomicBatchDispatcher(std::move(dispatchFunc));
+ createJobs(atomicBatchDispatcher, jobs, COUNT);
+ dispatchJobs(executor, jobs, results, DispatchProblem::PreprocessThrows, 8);
+ atomicBatchDispatcher.commit();
+ evb.loop();
+ validateResults<ABDTokenNotDispatchedException>(results, COUNT - 1);
+}
+
+TEST(FiberManager, ABD_MultipleDispatchOnSameTokenErrorTest) {
+ SET_UP_TEST_FUNC;
+
+ //
+ // Testing that calling dispatch more than once on the same token throws
+ //
+ dispatchFunc = userDispatchFunc;
+ auto atomicBatchDispatcher =
+ createAtomicBatchDispatcher(std::move(dispatchFunc));
+ createJobs(atomicBatchDispatcher, jobs, COUNT);
+ dispatchJobs(executor, jobs, results, DispatchProblem::DuplicateDispatch, 4);
+ atomicBatchDispatcher.commit();
+ evb.loop();
+}
+
+TEST(FiberManager, ABD_GetTokenCalledAfterCommitTest) {
+ SET_UP_TEST_FUNC;
+
+ //
+ // Testing that exception set on attempt to call getToken after commit called
+ //
+ dispatchFunc = userDispatchFunc;
+ auto atomicBatchDispatcher =
+ createAtomicBatchDispatcher(std::move(dispatchFunc));
+ createJobs(atomicBatchDispatcher, jobs, COUNT);
+ atomicBatchDispatcher.commit();
+ EXPECT_THROW(atomicBatchDispatcher.getToken(), ABDUsageException);
+ dispatchJobs(executor, jobs, results);
+ EXPECT_THROW(atomicBatchDispatcher.getToken(), ABDUsageException);
+ evb.loop();
+ validateResults(results, COUNT);
+ EXPECT_THROW(atomicBatchDispatcher.getToken(), ABDUsageException);
+}
+
+TEST(FiberManager, ABD_UserProvidedBatchDispatchThrowsTest) {
+ SET_UP_TEST_FUNC;
+
+ //
+ // Testing that exception is set if user provided batch dispatch throws
+ //
+ dispatchFunc = [](std::vector<ValueT>&& inputs) -> std::vector<ResultT> {
+ (void)userDispatchFunc(std::move(inputs));
+ throw std::runtime_error("Unexpected exception in user dispatch function");
+ };
+ auto atomicBatchDispatcher =
+ createAtomicBatchDispatcher(std::move(dispatchFunc));
+ createJobs(atomicBatchDispatcher, jobs, COUNT);
+ dispatchJobs(executor, jobs, results);
+ atomicBatchDispatcher.commit();
+ evb.loop();
+ validateResults<std::runtime_error>(results, COUNT);
+}
+
+TEST(FiberManager, VirtualEventBase) {
+ bool done1{false};
+ bool done2{false};
+ {
+ folly::ScopedEventBaseThread thread;
+
+ auto evb1 =
+ folly::make_unique<folly::VirtualEventBase>(*thread.getEventBase());
+ auto& evb2 = thread.getEventBase()->getVirtualEventBase();
+
+ getFiberManager(*evb1).addTaskRemote([&] {
+ Baton baton;
+ baton.timed_wait(std::chrono::milliseconds{100});
+
+ done1 = true;
+ });
+
+ getFiberManager(evb2).addTaskRemote([&] {
+ Baton baton;
+ baton.timed_wait(std::chrono::milliseconds{200});
+
+ done2 = true;
+ });
+
+ EXPECT_FALSE(done1);
+ EXPECT_FALSE(done2);
+
+ evb1.reset();
+ EXPECT_TRUE(done1);
+ EXPECT_FALSE(done2);
+ }
+ EXPECT_TRUE(done2);
+}
+
+TEST(TimedMutex, ThreadFiberDeadlockOrder) {
+ folly::EventBase evb;
+ auto& fm = getFiberManager(evb);
+ TimedMutex mutex;
+
+ mutex.lock();
+ std::thread unlockThread([&] {
+ /* sleep override */ std::this_thread::sleep_for(
+ std::chrono::milliseconds{100});
+ mutex.unlock();
+ });
+
+ fm.addTask([&] { std::lock_guard<TimedMutex> lg(mutex); });
+ fm.addTask([&] {
+ runInMainContext([&] {
+ auto locked = mutex.timed_lock(std::chrono::seconds{1});
+ EXPECT_TRUE(locked);
+ if (locked) {
+ mutex.unlock();
+ }
+ });
+ });
+
+ evb.loopOnce();
+ EXPECT_EQ(0, fm.hasTasks());
+
+ unlockThread.join();
+}
+
+TEST(TimedMutex, ThreadFiberDeadlockRace) {
+ folly::EventBase evb;
+ auto& fm = getFiberManager(evb);
+ TimedMutex mutex;
+
+ mutex.lock();
+
+ fm.addTask([&] {
+ auto locked = mutex.timed_lock(std::chrono::seconds{1});
+ EXPECT_TRUE(locked);
+ if (locked) {
+ mutex.unlock();
+ }
+ });
+ fm.addTask([&] {
+ mutex.unlock();
+ runInMainContext([&] {
+ auto locked = mutex.timed_lock(std::chrono::seconds{1});
+ EXPECT_TRUE(locked);
+ if (locked) {
+ mutex.unlock();
+ }
+ });
+ });
+
+ evb.loopOnce();
+ EXPECT_EQ(0, fm.hasTasks());
+}
+
/**
* Test that we can properly track fiber stack usage.
*
auto& loopController =
dynamic_cast<SimpleLoopController&>(fm.loopController());
- constexpr size_t n = 1000;
+ static constexpr size_t n = 1000;
int s = 0;
fm.addTask([&]() {
int b[n] = {0};
projects / folly.git / blobdiff