[lib/Fuzzer] use -fsanitize-coverage=trace-cmp when building LLVM with LLVM_USE_SANITIZE_COVERAGE; in lib/Fuzzer try to reload the corpus to pick up new units from other processes

llvm-svn: 236906

cmake/modules/HandleLLVMOptions.cmake

@@ -476,7 +476,7 @@ if(LLVM_USE_SANITIZER)
     message(WARNING "LLVM_USE_SANITIZER is not supported on this platform.")
   endif()
   if (LLVM_USE_SANITIZE_COVERAGE)
-    append("-fsanitize-coverage=edge,indirect-calls,8bit-counters" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
+    append("-fsanitize-coverage=edge,indirect-calls,8bit-counters,trace-cmp" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
   endif()
 endif()
 

lib/Fuzzer/FuzzerDriver.cpp

@@ -207,6 +207,7 @@ int FuzzerDriver(int argc, char **argv, UserCallback Callback) {
   Options.PreferSmallDuringInitialShuffle =
       Flags.prefer_small_during_initial_shuffle;
   Options.Tokens = ReadTokensFile(Flags.tokens);
+  Options.Reload = Flags.reload;
   if (Flags.runs >= 0)
     Options.MaxNumberOfRuns = Flags.runs;
   if (!inputs.empty())
@@ -235,8 +236,10 @@ int FuzzerDriver(int argc, char **argv, UserCallback Callback) {
   if (Flags.apply_tokens)
     return ApplyTokens(F, Flags.apply_tokens);
 
+  F.RereadOutputCorpus();
   for (auto &inp : inputs)
-    F.ReadDir(inp);
+    if (inp != Options.OutputCorpus)
+      F.ReadDir(inp, nullptr);
 
   if (F.CorpusSize() == 0)
     F.AddToCorpus(Unit());  // Can't fuzz empty corpus, so add an empty input.

lib/Fuzzer/FuzzerFlags.def

@@ -46,7 +46,9 @@ FUZZER_FLAG_INT(workers, 0,
             "Number of simultaneous worker processes to run the jobs.")
 FUZZER_FLAG_INT(dfsan, 1, "Use DFSan for taint-guided mutations. No-op unless "
                            "the DFSan instrumentation was compiled in.")
-
+FUZZER_FLAG_INT(reload, 1,
+                "Reload the main corpus periodically to get new units"
+                "discovered by other processes.")
 FUZZER_FLAG_STRING(tokens, "Use the file with tokens (one token per line) to"
                            " fuzz a token based input language.")
 FUZZER_FLAG_STRING(apply_tokens, "Read the given input file, substitute bytes "

lib/Fuzzer/FuzzerIO.cpp

@@ -13,10 +13,26 @@
 #include <iterator>
 #include <fstream>
 #include <dirent.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
 namespace fuzzer {
 
-static std::vector<std::string> ListFilesInDir(const std::string &Dir) {
+static long GetEpoch(const std::string &Path) {
+  struct stat St;
+  if (stat(Path.c_str(), &St)) return 0;
+  return St.st_mtime;
+}
+
+static std::vector<std::string> ListFilesInDir(const std::string &Dir,
+                                               long *Epoch) {
   std::vector<std::string> V;
+  if (Epoch) {
+    auto E = GetEpoch(Dir.c_str());
+    if (*Epoch >= E) return V;
+    *Epoch = E;
+  }
   DIR *D = opendir(Dir.c_str());
   if (!D) return V;
   while (auto E = readdir(D)) {
@@ -50,9 +66,14 @@ void WriteToFile(const Unit &U, const std::string &Path) {
   OF.write((const char*)U.data(), U.size());
 }
 
-void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V) {
+void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
-  for (auto &X : ListFilesInDir(Path))
+                            long *Epoch) {
-    V->push_back(FileToVector(DirPlusFile(Path, X)));
+  long E = Epoch ? *Epoch : 0;
+  for (auto &X : ListFilesInDir(Path, Epoch)) {
+    auto FilePath = DirPlusFile(Path, X);
+    if (Epoch && GetEpoch(FilePath) < E) continue;
+    V->push_back(FileToVector(FilePath));
+  }
 }
 
 std::string DirPlusFile(const std::string &DirPath,

lib/Fuzzer/FuzzerInternal.h

@@ -16,6 +16,7 @@
 #include <string>
 #include <vector>
 #include <unordered_set>
+#include <set>
 
 #include "FuzzerInterface.h"
 
@@ -25,7 +26,8 @@ using namespace std::chrono;
 
 std::string FileToString(const std::string &Path);
 Unit FileToVector(const std::string &Path);
-void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V);
+void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
+                            long *Epoch);
 void WriteToFile(const Unit &U, const std::string &Path);
 void CopyFileToErr(const std::string &Path);
 // Returns "Dir/FileName" or equivalent for the current OS.
@@ -54,6 +56,7 @@ class Fuzzer {
     bool UseFullCoverageSet  = false;
     bool UseCoveragePairs = false;
     bool UseDFSan = false;
+    bool Reload = true;
     int PreferSmallDuringInitialShuffle = -1;
     size_t MaxNumberOfRuns = ULONG_MAX;
     std::string OutputCorpus;
@@ -65,9 +68,10 @@ class Fuzzer {
   void ShuffleAndMinimize();
   void InitializeDFSan();
   size_t CorpusSize() const { return Corpus.size(); }
-  void ReadDir(const std::string &Path) {
+  void ReadDir(const std::string &Path, long *Epoch) {
-    ReadDirToVectorOfUnits(Path.c_str(), &Corpus);
+    ReadDirToVectorOfUnits(Path.c_str(), &Corpus, Epoch);
   }
+  void RereadOutputCorpus();
   // Save the current corpus to OutputCorpus.
   void SaveCorpus();
 
@@ -116,6 +120,7 @@ class Fuzzer {
   size_t TotalNumberOfRuns = 0;
 
   std::vector<Unit> Corpus;
+  std::set<Unit> UnitsAddedAfterInitialLoad;
   std::unordered_set<uintptr_t> FullCoverageSets;
   std::unordered_set<uint64_t>  CoveragePairs;
 
@@ -132,6 +137,7 @@ class Fuzzer {
   system_clock::time_point ProcessStartTime = system_clock::now();
   system_clock::time_point UnitStartTime;
   long TimeOfLongestUnitInSeconds = 0;
+  long EpochOfLastReadOfOutputCorpus = 0;
 };
 
 };  // namespace fuzzer

lib/Fuzzer/FuzzerLoop.cpp

@@ -86,6 +86,30 @@ void Fuzzer::PrintStats(const char *Where, size_t Cov, const char *End) {
       << End;
 }
 
+void Fuzzer::RereadOutputCorpus() {
+  if (Options.OutputCorpus.empty()) return;
+  std::vector<Unit> AdditionalCorpus;
+  ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
+                         &EpochOfLastReadOfOutputCorpus);
+  if (Corpus.empty()) {
+    Corpus = AdditionalCorpus;
+    return;
+  }
+  if (!Options.Reload) return;
+  for (auto &X : AdditionalCorpus) {
+    if (X.size() > (size_t)Options.MaxLen)
+      X.resize(Options.MaxLen);
+    if (UnitsAddedAfterInitialLoad.insert(X).second) {
+      Corpus.push_back(X);
+      CurrentUnit.clear();
+      CurrentUnit.insert(CurrentUnit.begin(), X.begin(), X.end());
+      size_t NewCoverage = RunOne(CurrentUnit);
+      if (NewCoverage && Options.Verbosity >= 1)
+        PrintStats("RELOAD", NewCoverage);
+    }
+  }
+}
+
 void Fuzzer::ShuffleAndMinimize() {
   size_t MaxCov = 0;
   bool PreferSmall =
@@ -268,6 +292,7 @@ void Fuzzer::SaveCorpus() {
 void Fuzzer::ReportNewCoverage(size_t NewCoverage, const Unit &U) {
   if (!NewCoverage) return;
   Corpus.push_back(U);
+  UnitsAddedAfterInitialLoad.insert(U);
   PrintStats("NEW   ", NewCoverage, "");
   if (Options.Verbosity) {
     std::cerr << " L: " << U.size();
@@ -299,6 +324,7 @@ void Fuzzer::MutateAndTestOne(Unit *U) {
 void Fuzzer::Loop(size_t NumIterations) {
   for (size_t i = 1; i <= NumIterations; i++) {
     for (size_t J1 = 0; J1 < Corpus.size(); J1++) {
+      RereadOutputCorpus();
       if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
         return;
       // First, simply mutate the unit w/o doing crosses.

lib/Fuzzer/test/dfsan/CMakeLists.txt

@@ -1,7 +1,7 @@
 # These tests depend on both coverage and dfsan instrumentation.
 
 set(CMAKE_CXX_FLAGS_RELEASE
-  "${LIBFUZZER_FLAGS_BASE} -O0 -fno-sanitize=all -fsanitize=dataflow -mllvm -sanitizer-coverage-experimental-trace-compares=1")
+  "${LIBFUZZER_FLAGS_BASE} -O0 -fno-sanitize=all -fsanitize=dataflow")
 
 foreach(Test ${DFSanTests})
   add_executable(LLVMFuzzer-${Test}