DIRECTORY experimental/logging/test/
TEST async_file_writer_test SOURCES AsyncFileWriterTest.cpp
TEST config_parser_test SOURCES ConfigParserTest.cpp
+ TEST file_handler_factory_test SOURCES FileHandlerFactoryTest.cpp
TEST glog_formatter_test SOURCES GlogFormatterTest.cpp
TEST immediate_file_writer_test SOURCES ImmediateFileWriterTest.cpp
TEST log_category_test SOURCES LogCategoryTest.cpp
experimental/JSONSchema.h \
experimental/LockFreeRingBuffer.h \
experimental/logging/AsyncFileWriter.h \
+ experimental/logging/FileHandlerFactory.h \
experimental/logging/GlogStyleFormatter.h \
experimental/logging/ImmediateFileWriter.h \
experimental/logging/Init.h \
experimental/logging/Logger.h \
experimental/logging/LoggerDB.h \
experimental/logging/LogHandler.h \
+ experimental/logging/LogHandlerFactory.h \
experimental/logging/LogHandlerConfig.h \
experimental/logging/LogLevel.h \
experimental/logging/LogMessage.h \
namespace folly {
+constexpr size_t AsyncFileWriter::kDefaultMaxBufferSize;
+
AsyncFileWriter::AsyncFileWriter(StringPiece path)
: AsyncFileWriter{File{path.str(), O_WRONLY | O_APPEND | O_CREAT}} {}
}
}
+void AsyncFileWriter::setMaxBufferSize(size_t size) {
+ auto data = data_.lock();
+ data->maxBufferBytes = size;
+}
+
+size_t AsyncFileWriter::getMaxBufferSize() const {
+ auto data = data_.lock();
+ return data->maxBufferBytes;
+}
+
void AsyncFileWriter::ioThread() {
folly::setThreadName("log_writer");
*/
class AsyncFileWriter : public LogWriter {
public:
+ /**
+ * The default maximum buffer size.
+ *
+ * The comments for setMaxBufferSize() explain how this parameter is used.
+ */
+ static constexpr size_t kDefaultMaxBufferSize = 1024 * 1024;
+
/**
* Construct an AsyncFileWriter that appends to the file at the specified
* path.
*/
void flush() override;
+ /**
+ * Set the maximum buffer size for this AsyncFileWriter, in bytes.
+ *
+ * This controls the upper bound on how much unwritten data will be buffered
+ * in memory. If messages are being logged faster than they can be written
+ * to output file, new messages will be discarded if they would cause the
+ * amount of buffered data to exceed this limit.
+ */
+ void setMaxBufferSize(size_t size);
+
+ /**
+ * Get the maximum buffer size for this AsyncFileWriter, in bytes.
+ */
+ size_t getMaxBufferSize() const;
+
+ /**
+ * Get the output file.
+ */
+ const folly::File& getFile() const {
+ return file_;
+ }
+
private:
/*
* A simple implementation using two queues.
bool stop{false};
bool ioThreadDone{false};
uint64_t ioThreadCounter{0};
- size_t maxBufferBytes{1024 * 1024};
+ size_t maxBufferBytes{kDefaultMaxBufferSize};
size_t currentBufferSize{0};
size_t numDiscarded{0};
--- /dev/null
+/*
+ * Copyright 2004-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <folly/experimental/logging/FileHandlerFactory.h>
+
+#include <set>
+
+#include <folly/Conv.h>
+#include <folly/MapUtil.h>
+#include <folly/experimental/logging/AsyncFileWriter.h>
+#include <folly/experimental/logging/GlogStyleFormatter.h>
+#include <folly/experimental/logging/ImmediateFileWriter.h>
+#include <folly/experimental/logging/StandardLogHandler.h>
+
+using std::make_shared;
+using std::shared_ptr;
+using std::string;
+
+namespace folly {
+
+std::shared_ptr<LogHandler> FileHandlerFactory::createHandler(
+ const Options& options) {
+ // Raise an error if we receive unexpected options
+ auto knownOptions =
+ std::set<std::string>{"path", "stream", "async", "max_buffer_size"};
+ for (const auto& entry : options) {
+ if (knownOptions.find(entry.first) == knownOptions.end()) {
+ throw std::invalid_argument(to<string>(
+ "unknown parameter \"", entry.first, "\" for FileHandlerFactory"));
+ }
+ }
+
+ // Construct the formatter
+ // TODO: We should eventually support parameters to control the formatter
+ // behavior.
+ auto formatter = make_shared<GlogStyleFormatter>();
+
+ // Get the output file to use
+ File outputFile;
+ auto* path = get_ptr(options, "path");
+ auto* stream = get_ptr(options, "stream");
+ if (path && stream) {
+ throw std::invalid_argument(
+ "cannot specify both \"path\" and \"stream\" "
+ "parameters for FileHandlerFactory");
+ } else if (path) {
+ outputFile = File{*path, O_WRONLY | O_APPEND | O_CREAT};
+ } else if (stream) {
+ if (*stream == "stderr") {
+ outputFile = File{STDERR_FILENO, /* ownsFd */ false};
+ } else if (*stream == "stdout") {
+ outputFile = File{STDOUT_FILENO, /* ownsFd */ false};
+ } else {
+ throw std::invalid_argument(to<string>(
+ "unknown stream for FileHandlerFactory: \"",
+ *stream,
+ "\" expected one of stdout or stderr"));
+ }
+ } else {
+ throw std::invalid_argument(
+ "must specify a \"path\" or \"stream\" "
+ "parameter for FileHandlerFactory");
+ }
+
+ // Determine whether we should use ImmediateFileWriter or AsyncFileWriter
+ shared_ptr<LogWriter> writer;
+ bool async = true;
+ auto* asyncOption = get_ptr(options, "async");
+ if (asyncOption) {
+ try {
+ async = to<bool>(*asyncOption);
+ } catch (const std::exception& ex) {
+ throw std::invalid_argument(to<string>(
+ "expected a boolean value for FileHandlerFactory \"async\" "
+ "parameter: ",
+ *asyncOption));
+ }
+ }
+ auto* maxBufferOption = get_ptr(options, "max_buffer_size");
+ if (async) {
+ auto asyncWriter = make_shared<AsyncFileWriter>(std::move(outputFile));
+ if (maxBufferOption) {
+ size_t maxBufferSize;
+ try {
+ maxBufferSize = to<size_t>(*maxBufferOption);
+ } catch (const std::exception& ex) {
+ throw std::invalid_argument(to<string>(
+ "expected an integer value for FileHandlerFactory "
+ "\"max_buffer_size\": ",
+ *maxBufferOption));
+ }
+ if (maxBufferSize == 0) {
+ throw std::invalid_argument(to<string>(
+ "expected a positive value for FileHandlerFactory "
+ "\"max_buffer_size\": ",
+ *maxBufferOption));
+ }
+ asyncWriter->setMaxBufferSize(maxBufferSize);
+ }
+ writer = std::move(asyncWriter);
+ } else {
+ if (maxBufferOption) {
+ throw std::invalid_argument(to<string>(
+ "the \"max_buffer_size\" option is only valid for async file "
+ "handlers"));
+ }
+ writer = make_shared<ImmediateFileWriter>(std::move(outputFile));
+ }
+
+ return make_shared<StandardLogHandler>(formatter, writer);
+}
+
+} // namespace folly
--- /dev/null
+/*
+ * Copyright 2004-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <folly/experimental/logging/LogHandlerFactory.h>
+
+namespace folly {
+
+/**
+ * FileHandlerFactory is a LogHandlerFactory that constructs log handlers
+ * that write to a file.
+ *
+ * It can construct handlers that use either ImmediateFileWriter or
+ * AsyncFileWriter.
+ */
+class FileHandlerFactory : public LogHandlerFactory {
+ public:
+ StringPiece getType() const override {
+ return "file";
+ }
+
+ std::shared_ptr<LogHandler> createHandler(const Options& options) override;
+};
+
+} // namespace folly
void writeMessage(folly::StringPiece buffer, uint32_t flags = 0) override;
void flush() override;
+ /**
+ * Get the output file.
+ */
+ const folly::File& getFile() const {
+ return file_;
+ }
+
private:
ImmediateFileWriter(ImmediateFileWriter const&) = delete;
ImmediateFileWriter& operator=(ImmediateFileWriter const&) = delete;
--- /dev/null
+/*
+ * Copyright 2004-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+#include <folly/CppAttributes.h>
+#include <folly/Range.h>
+
+namespace folly {
+
+class LogHandler;
+
+class LogHandlerFactory {
+ public:
+ using Options = std::unordered_map<std::string, std::string>;
+
+ virtual ~LogHandlerFactory() = default;
+
+ /**
+ * Get the type name of this LogHandlerFactory.
+ *
+ * The type field in the LogHandlerConfig for all LogHandlers created by this
+ * factory should match the type of the LogHandlerFactory.
+ *
+ * The type of a LogHandlerFactory should never change. The returned
+ * StringPiece should be valid for the lifetime of the LogHandlerFactory.
+ */
+ virtual StringPiece getType() const = 0;
+
+ /**
+ * Create a new LogHandler.
+ */
+ virtual std::shared_ptr<LogHandler> createHandler(const Options& options) = 0;
+
+ /**
+ * Update an existing LogHandler with a new configuration.
+ *
+ * This may create a new LogHandler object, or it may update the existing
+ * LogHandler in place.
+ *
+ * The returned pointer will point to the input handler if it was updated in
+ * place, or will point to a new LogHandler if a new one was created.
+ */
+ virtual std::shared_ptr<LogHandler> updateHandler(
+ FOLLY_MAYBE_UNUSED const std::shared_ptr<LogHandler>& existingHandler,
+ const Options& options) {
+ // Subclasses may override this with functionality to update an existing
+ // handler in-place. However, provide a default implementation that simply
+ // calls createHandler() to always create a new handler object.
+ return createHandler(options);
+ }
+};
+
+} // namespace folly
libfollylogging_la_SOURCES = \
AsyncFileWriter.cpp \
+ FileHandlerFactory.cpp \
GlogStyleFormatter.cpp \
ImmediateFileWriter.cpp \
Init.cpp \
std::shared_ptr<LogWriter> writer);
~StandardLogHandler();
+ /**
+ * Get the LogFormatter used by this handler.
+ */
+ const std::shared_ptr<LogFormatter>& getFormatter() const {
+ return formatter_;
+ }
+
+ /**
+ * Get the LogWriter used by this handler.
+ */
+ const std::shared_ptr<LogWriter>& getWriter() const {
+ return writer_;
+ }
+
/**
* Get the handler's current LogLevel.
*
private:
std::atomic<LogLevel> level_{LogLevel::NONE};
- std::shared_ptr<LogFormatter> formatter_;
- std::shared_ptr<LogWriter> writer_;
+
+ // The formatter_ and writer_ member variables are const, and cannot be
+ // modified after the StandardLogHandler is constructed. This allows them to
+ // be accessed without locking when handling a message. To change these
+ // values, create a new StandardLogHandler object and replace the old handler
+ // with the new one in the LoggerDB.
+
+ const std::shared_ptr<LogFormatter> formatter_;
+ const std::shared_ptr<LogWriter> writer_;
};
} // namespace folly
--- /dev/null
+/*
+ * Copyright 2004-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <folly/experimental/logging/FileHandlerFactory.h>
+
+#include <folly/Exception.h>
+#include <folly/experimental/TestUtil.h>
+#include <folly/experimental/logging/AsyncFileWriter.h>
+#include <folly/experimental/logging/GlogStyleFormatter.h>
+#include <folly/experimental/logging/ImmediateFileWriter.h>
+#include <folly/experimental/logging/StandardLogHandler.h>
+#include <folly/portability/GTest.h>
+
+using namespace folly;
+using folly::test::TemporaryFile;
+using std::make_pair;
+
+void checkAsyncWriter(
+ const LogWriter* writer,
+ const char* expectedPath,
+ size_t expectedMaxBufferSize) {
+ auto asyncWriter = dynamic_cast<const AsyncFileWriter*>(writer);
+ ASSERT_TRUE(asyncWriter)
+ << "FileHandlerFactory should have created an AsyncFileWriter";
+ EXPECT_EQ(expectedMaxBufferSize, asyncWriter->getMaxBufferSize());
+
+ // Make sure this refers to the expected output file
+ struct stat expectedStatInfo;
+ checkUnixError(stat(expectedPath, &expectedStatInfo), "stat failed");
+ struct stat actualStatInfo;
+ checkUnixError(
+ fstat(asyncWriter->getFile().fd(), &actualStatInfo), "fstat failed");
+ EXPECT_EQ(expectedStatInfo.st_dev, actualStatInfo.st_dev);
+ EXPECT_EQ(expectedStatInfo.st_ino, actualStatInfo.st_ino);
+}
+
+void checkAsyncWriter(
+ const LogWriter* writer,
+ int expectedFD,
+ size_t expectedMaxBufferSize) {
+ auto asyncWriter = dynamic_cast<const AsyncFileWriter*>(writer);
+ ASSERT_TRUE(asyncWriter)
+ << "FileHandlerFactory should have created an AsyncFileWriter";
+ EXPECT_EQ(expectedMaxBufferSize, asyncWriter->getMaxBufferSize());
+ EXPECT_EQ(expectedFD, asyncWriter->getFile().fd());
+}
+
+TEST(FileHandlerFactory, pathOnly) {
+ FileHandlerFactory factory;
+
+ TemporaryFile tmpFile{"logging_test"};
+ auto options = FileHandlerFactory::Options{
+ make_pair("path", tmpFile.path().string()),
+ };
+ auto handler = factory.createHandler(options);
+
+ auto stdHandler = std::dynamic_pointer_cast<StandardLogHandler>(handler);
+ ASSERT_TRUE(stdHandler);
+
+ auto formatter =
+ std::dynamic_pointer_cast<GlogStyleFormatter>(stdHandler->getFormatter());
+ EXPECT_TRUE(formatter)
+ << "FileHandlerFactory should have created a GlogStyleFormatter";
+
+ checkAsyncWriter(
+ stdHandler->getWriter().get(),
+ tmpFile.path().string().c_str(),
+ AsyncFileWriter::kDefaultMaxBufferSize);
+}
+
+TEST(FileHandlerFactory, stderrStream) {
+ FileHandlerFactory factory;
+
+ TemporaryFile tmpFile{"logging_test"};
+ auto options = FileHandlerFactory::Options{
+ make_pair("stream", "stderr"),
+ };
+ auto handler = factory.createHandler(options);
+
+ auto stdHandler = std::dynamic_pointer_cast<StandardLogHandler>(handler);
+ ASSERT_TRUE(stdHandler);
+
+ auto formatter =
+ std::dynamic_pointer_cast<GlogStyleFormatter>(stdHandler->getFormatter());
+ EXPECT_TRUE(formatter)
+ << "FileHandlerFactory should have created a GlogStyleFormatter";
+
+ checkAsyncWriter(
+ stdHandler->getWriter().get(),
+ STDERR_FILENO,
+ AsyncFileWriter::kDefaultMaxBufferSize);
+}
+
+TEST(FileHandlerFactory, stdoutWithMaxBuffer) {
+ FileHandlerFactory factory;
+
+ TemporaryFile tmpFile{"logging_test"};
+ auto options = FileHandlerFactory::Options{
+ make_pair("stream", "stdout"),
+ make_pair("max_buffer_size", "4096"),
+ };
+ auto handler = factory.createHandler(options);
+
+ auto stdHandler = std::dynamic_pointer_cast<StandardLogHandler>(handler);
+ ASSERT_TRUE(stdHandler);
+
+ auto formatter =
+ std::dynamic_pointer_cast<GlogStyleFormatter>(stdHandler->getFormatter());
+ EXPECT_TRUE(formatter)
+ << "FileHandlerFactory should have created a GlogStyleFormatter";
+
+ checkAsyncWriter(stdHandler->getWriter().get(), STDOUT_FILENO, 4096);
+}
+
+TEST(FileHandlerFactory, pathWithMaxBufferSize) {
+ FileHandlerFactory factory;
+
+ TemporaryFile tmpFile{"logging_test"};
+ auto options = FileHandlerFactory::Options{
+ make_pair("path", tmpFile.path().string()),
+ make_pair("max_buffer_size", "4096000"),
+ };
+ auto handler = factory.createHandler(options);
+
+ auto stdHandler = std::dynamic_pointer_cast<StandardLogHandler>(handler);
+ ASSERT_TRUE(stdHandler);
+
+ auto formatter =
+ std::dynamic_pointer_cast<GlogStyleFormatter>(stdHandler->getFormatter());
+ EXPECT_TRUE(formatter)
+ << "FileHandlerFactory should have created a GlogStyleFormatter";
+
+ checkAsyncWriter(
+ stdHandler->getWriter().get(), tmpFile.path().string().c_str(), 4096000);
+}
+
+TEST(FileHandlerFactory, nonAsyncStderr) {
+ FileHandlerFactory factory;
+
+ TemporaryFile tmpFile{"logging_test"};
+ auto options = FileHandlerFactory::Options{
+ make_pair("stream", "stderr"),
+ make_pair("async", "no"),
+ };
+ auto handler = factory.createHandler(options);
+
+ auto stdHandler = std::dynamic_pointer_cast<StandardLogHandler>(handler);
+ ASSERT_TRUE(stdHandler);
+
+ auto formatter =
+ std::dynamic_pointer_cast<GlogStyleFormatter>(stdHandler->getFormatter());
+ EXPECT_TRUE(formatter)
+ << "FileHandlerFactory should have created a GlogStyleFormatter";
+
+ auto writer =
+ std::dynamic_pointer_cast<ImmediateFileWriter>(stdHandler->getWriter());
+ ASSERT_TRUE(writer);
+ EXPECT_EQ(STDERR_FILENO, writer->getFile().fd());
+}
+
+TEST(FileHandlerFactory, errors) {
+ FileHandlerFactory factory;
+ TemporaryFile tmpFile{"logging_test"};
+
+ {
+ auto options = FileHandlerFactory::Options{};
+ EXPECT_THROW(factory.createHandler(options), std::invalid_argument)
+ << "one of path or stream required";
+ }
+
+ {
+ auto options = FileHandlerFactory::Options{
+ make_pair("path", tmpFile.path().string()),
+ make_pair("stream", "stderr"),
+ };
+ EXPECT_THROW(factory.createHandler(options), std::invalid_argument)
+ << "path and stream cannot both be specified";
+ }
+
+ {
+ auto options = FileHandlerFactory::Options{
+ make_pair("stream", "nonstdout"),
+ };
+ EXPECT_THROW(factory.createHandler(options), std::invalid_argument)
+ << "invalid stream";
+ }
+
+ {
+ auto options = FileHandlerFactory::Options{
+ make_pair("stream", "stderr"),
+ make_pair("async", "foobar"),
+ };
+ EXPECT_THROW(factory.createHandler(options), std::invalid_argument)
+ << "invalid async value";
+ }
+
+ {
+ auto options = FileHandlerFactory::Options{
+ make_pair("stream", "stderr"),
+ make_pair("async", "false"),
+ make_pair("max_buffer_size", "1234"),
+ };
+ EXPECT_THROW(factory.createHandler(options), std::invalid_argument)
+ << "max_buffer_size only valid for async writers";
+ }
+
+ {
+ auto options = FileHandlerFactory::Options{
+ make_pair("stream", "stderr"),
+ make_pair("max_buffer_size", "hello"),
+ };
+ EXPECT_THROW(factory.createHandler(options), std::invalid_argument)
+ << "max_buffer_size must be an integer";
+ }
+
+ {
+ auto options = FileHandlerFactory::Options{
+ make_pair("stream", "stderr"),
+ make_pair("max_buffer_size", "0"),
+ };
+ EXPECT_THROW(factory.createHandler(options), std::invalid_argument)
+ << "max_buffer_size must be a positive integer";
+ }
+
+ {
+ auto options = FileHandlerFactory::Options{
+ make_pair("stream", "stderr"),
+ make_pair("foo", "bar"),
+ };
+ EXPECT_THROW(factory.createHandler(options), std::invalid_argument)
+ << "unknown parameter foo";
+ }
+}
projects / folly.git / commitdiff