From 8ae273d380e3630921762ae32aff39271134e7e7 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Fri, 8 May 2015 21:30:55 +0000 Subject: [PATCH] [lib/Fuzzer] use -fsanitize-coverage=trace-cmp when building LLVM with LLVM_USE_SANITIZE_COVERAGE; in lib/Fuzzer try to reload the corpus to pick up new units from other processes git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@236906 91177308-0d34-0410-b5e6-96231b3b80d8 --- cmake/modules/HandleLLVMOptions.cmake | 2 +- lib/Fuzzer/FuzzerDriver.cpp | 5 ++++- lib/Fuzzer/FuzzerFlags.def | 4 +++- lib/Fuzzer/FuzzerIO.cpp | 29 +++++++++++++++++++++++---- lib/Fuzzer/FuzzerInternal.h | 12 ++++++++--- lib/Fuzzer/FuzzerLoop.cpp | 26 ++++++++++++++++++++++++ lib/Fuzzer/test/dfsan/CMakeLists.txt | 2 +- 7 files changed, 69 insertions(+), 11 deletions(-) diff --git a/cmake/modules/HandleLLVMOptions.cmake b/cmake/modules/HandleLLVMOptions.cmake index b0da64d9745..d2b89748765 100644 --- a/cmake/modules/HandleLLVMOptions.cmake +++ b/cmake/modules/HandleLLVMOptions.cmake @@ -476,7 +476,7 @@ if(LLVM_USE_SANITIZER) message(WARNING "LLVM_USE_SANITIZER is not supported on this platform.") endif() if (LLVM_USE_SANITIZE_COVERAGE) - append("-fsanitize-coverage=edge,indirect-calls,8bit-counters" CMAKE_C_FLAGS CMAKE_CXX_FLAGS) + append("-fsanitize-coverage=edge,indirect-calls,8bit-counters,trace-cmp" CMAKE_C_FLAGS CMAKE_CXX_FLAGS) endif() endif() diff --git a/lib/Fuzzer/FuzzerDriver.cpp b/lib/Fuzzer/FuzzerDriver.cpp index 05a699e5068..641221194c5 100644 --- a/lib/Fuzzer/FuzzerDriver.cpp +++ b/lib/Fuzzer/FuzzerDriver.cpp @@ -207,6 +207,7 @@ int FuzzerDriver(int argc, char **argv, UserCallback Callback) { Options.PreferSmallDuringInitialShuffle = Flags.prefer_small_during_initial_shuffle; Options.Tokens = ReadTokensFile(Flags.tokens); + Options.Reload = Flags.reload; if (Flags.runs >= 0) Options.MaxNumberOfRuns = Flags.runs; if (!inputs.empty()) @@ -235,8 +236,10 @@ int FuzzerDriver(int argc, char **argv, UserCallback Callback) { if (Flags.apply_tokens) return ApplyTokens(F, Flags.apply_tokens); + F.RereadOutputCorpus(); for (auto &inp : inputs) - F.ReadDir(inp); + if (inp != Options.OutputCorpus) + F.ReadDir(inp, nullptr); if (F.CorpusSize() == 0) F.AddToCorpus(Unit()); // Can't fuzz empty corpus, so add an empty input. diff --git a/lib/Fuzzer/FuzzerFlags.def b/lib/Fuzzer/FuzzerFlags.def index dbaf75db696..1222feb5937 100644 --- a/lib/Fuzzer/FuzzerFlags.def +++ b/lib/Fuzzer/FuzzerFlags.def @@ -46,7 +46,9 @@ FUZZER_FLAG_INT(workers, 0, "Number of simultaneous worker processes to run the jobs.") FUZZER_FLAG_INT(dfsan, 1, "Use DFSan for taint-guided mutations. No-op unless " "the DFSan instrumentation was compiled in.") - +FUZZER_FLAG_INT(reload, 1, + "Reload the main corpus periodically to get new units" + "discovered by other processes.") FUZZER_FLAG_STRING(tokens, "Use the file with tokens (one token per line) to" " fuzz a token based input language.") FUZZER_FLAG_STRING(apply_tokens, "Read the given input file, substitute bytes " diff --git a/lib/Fuzzer/FuzzerIO.cpp b/lib/Fuzzer/FuzzerIO.cpp index 81f37aa3c01..7136d38fb77 100644 --- a/lib/Fuzzer/FuzzerIO.cpp +++ b/lib/Fuzzer/FuzzerIO.cpp @@ -13,10 +13,26 @@ #include #include #include +#include +#include +#include + namespace fuzzer { -static std::vector ListFilesInDir(const std::string &Dir) { +static long GetEpoch(const std::string &Path) { + struct stat St; + if (stat(Path.c_str(), &St)) return 0; + return St.st_mtime; +} + +static std::vector ListFilesInDir(const std::string &Dir, + long *Epoch) { std::vector V; + if (Epoch) { + auto E = GetEpoch(Dir.c_str()); + if (*Epoch >= E) return V; + *Epoch = E; + } DIR *D = opendir(Dir.c_str()); if (!D) return V; while (auto E = readdir(D)) { @@ -50,9 +66,14 @@ void WriteToFile(const Unit &U, const std::string &Path) { OF.write((const char*)U.data(), U.size()); } -void ReadDirToVectorOfUnits(const char *Path, std::vector *V) { - for (auto &X : ListFilesInDir(Path)) - V->push_back(FileToVector(DirPlusFile(Path, X))); +void ReadDirToVectorOfUnits(const char *Path, std::vector *V, + long *Epoch) { + long E = Epoch ? *Epoch : 0; + for (auto &X : ListFilesInDir(Path, Epoch)) { + auto FilePath = DirPlusFile(Path, X); + if (Epoch && GetEpoch(FilePath) < E) continue; + V->push_back(FileToVector(FilePath)); + } } std::string DirPlusFile(const std::string &DirPath, diff --git a/lib/Fuzzer/FuzzerInternal.h b/lib/Fuzzer/FuzzerInternal.h index 0fead355531..8363ef020b2 100644 --- a/lib/Fuzzer/FuzzerInternal.h +++ b/lib/Fuzzer/FuzzerInternal.h @@ -16,6 +16,7 @@ #include #include #include +#include #include "FuzzerInterface.h" @@ -25,7 +26,8 @@ using namespace std::chrono; std::string FileToString(const std::string &Path); Unit FileToVector(const std::string &Path); -void ReadDirToVectorOfUnits(const char *Path, std::vector *V); +void ReadDirToVectorOfUnits(const char *Path, std::vector *V, + long *Epoch); void WriteToFile(const Unit &U, const std::string &Path); void CopyFileToErr(const std::string &Path); // Returns "Dir/FileName" or equivalent for the current OS. @@ -54,6 +56,7 @@ class Fuzzer { bool UseFullCoverageSet = false; bool UseCoveragePairs = false; bool UseDFSan = false; + bool Reload = true; int PreferSmallDuringInitialShuffle = -1; size_t MaxNumberOfRuns = ULONG_MAX; std::string OutputCorpus; @@ -65,9 +68,10 @@ class Fuzzer { void ShuffleAndMinimize(); void InitializeDFSan(); size_t CorpusSize() const { return Corpus.size(); } - void ReadDir(const std::string &Path) { - ReadDirToVectorOfUnits(Path.c_str(), &Corpus); + void ReadDir(const std::string &Path, long *Epoch) { + ReadDirToVectorOfUnits(Path.c_str(), &Corpus, Epoch); } + void RereadOutputCorpus(); // Save the current corpus to OutputCorpus. void SaveCorpus(); @@ -116,6 +120,7 @@ class Fuzzer { size_t TotalNumberOfRuns = 0; std::vector Corpus; + std::set UnitsAddedAfterInitialLoad; std::unordered_set FullCoverageSets; std::unordered_set CoveragePairs; @@ -132,6 +137,7 @@ class Fuzzer { system_clock::time_point ProcessStartTime = system_clock::now(); system_clock::time_point UnitStartTime; long TimeOfLongestUnitInSeconds = 0; + long EpochOfLastReadOfOutputCorpus = 0; }; }; // namespace fuzzer diff --git a/lib/Fuzzer/FuzzerLoop.cpp b/lib/Fuzzer/FuzzerLoop.cpp index 9d35384ecd5..9f9d92e2c0c 100644 --- a/lib/Fuzzer/FuzzerLoop.cpp +++ b/lib/Fuzzer/FuzzerLoop.cpp @@ -86,6 +86,30 @@ void Fuzzer::PrintStats(const char *Where, size_t Cov, const char *End) { << End; } +void Fuzzer::RereadOutputCorpus() { + if (Options.OutputCorpus.empty()) return; + std::vector AdditionalCorpus; + ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus, + &EpochOfLastReadOfOutputCorpus); + if (Corpus.empty()) { + Corpus = AdditionalCorpus; + return; + } + if (!Options.Reload) return; + for (auto &X : AdditionalCorpus) { + if (X.size() > (size_t)Options.MaxLen) + X.resize(Options.MaxLen); + if (UnitsAddedAfterInitialLoad.insert(X).second) { + Corpus.push_back(X); + CurrentUnit.clear(); + CurrentUnit.insert(CurrentUnit.begin(), X.begin(), X.end()); + size_t NewCoverage = RunOne(CurrentUnit); + if (NewCoverage && Options.Verbosity >= 1) + PrintStats("RELOAD", NewCoverage); + } + } +} + void Fuzzer::ShuffleAndMinimize() { size_t MaxCov = 0; bool PreferSmall = @@ -268,6 +292,7 @@ void Fuzzer::SaveCorpus() { void Fuzzer::ReportNewCoverage(size_t NewCoverage, const Unit &U) { if (!NewCoverage) return; Corpus.push_back(U); + UnitsAddedAfterInitialLoad.insert(U); PrintStats("NEW ", NewCoverage, ""); if (Options.Verbosity) { std::cerr << " L: " << U.size(); @@ -299,6 +324,7 @@ void Fuzzer::MutateAndTestOne(Unit *U) { void Fuzzer::Loop(size_t NumIterations) { for (size_t i = 1; i <= NumIterations; i++) { for (size_t J1 = 0; J1 < Corpus.size(); J1++) { + RereadOutputCorpus(); if (TotalNumberOfRuns >= Options.MaxNumberOfRuns) return; // First, simply mutate the unit w/o doing crosses. diff --git a/lib/Fuzzer/test/dfsan/CMakeLists.txt b/lib/Fuzzer/test/dfsan/CMakeLists.txt index f92fde9bcac..d1a140d6b50 100644 --- a/lib/Fuzzer/test/dfsan/CMakeLists.txt +++ b/lib/Fuzzer/test/dfsan/CMakeLists.txt @@ -1,7 +1,7 @@ # These tests depend on both coverage and dfsan instrumentation. set(CMAKE_CXX_FLAGS_RELEASE - "${LIBFUZZER_FLAGS_BASE} -O0 -fno-sanitize=all -fsanitize=dataflow -mllvm -sanitizer-coverage-experimental-trace-compares=1") + "${LIBFUZZER_FLAGS_BASE} -O0 -fno-sanitize=all -fsanitize=dataflow") foreach(Test ${DFSanTests}) add_executable(LLVMFuzzer-${Test} -- 2.34.1