Generate error reports when a fuzz target exits.

Summary:
Implements https://github.com/google/sanitizers/issues/835.

Flush stdout before exiting in test cases.

Since the atexit hook is used for exit reports, pending prints to
stdout can be lost if they aren't flushed before calling exit().

Expect tests to have non-zero exit code if exit() is called.

Reviewers: vitalybuka, kcc

Reviewed By: kcc

Subscribers: eraman, llvm-commits, hiraditya

Differential Revision: https://reviews.llvm.org/D35602

llvm-svn: 308669

lib/Fuzzer/FuzzerDriver.cpp

@@ -20,6 +20,7 @@
 #include <algorithm>
 #include <atomic>
 #include <chrono>
+#include <cstdlib>
 #include <cstring>
 #include <mutex>
 #include <string>
@@ -642,6 +643,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   Options.HandleXfsz = Flags.handle_xfsz;
   SetSignalHandler(Options);
 
+  std::atexit(Fuzzer::StaticExitCallback);
+
   if (Flags.minimize_crash)
     return MinimizeCrashInput(Args, Options);
 

lib/Fuzzer/FuzzerInternal.h

@@ -60,6 +60,7 @@ public:
 
   static void StaticAlarmCallback();
   static void StaticCrashSignalCallback();
+  static void StaticExitCallback();
   static void StaticInterruptCallback();
   static void StaticFileSizeExceedCallback();
 
@@ -91,6 +92,7 @@ public:
 private:
   void AlarmCallback();
   void CrashCallback();
+  void ExitCallback();
   void CrashOnOverwrittenData();
   void InterruptCallback();
   void MutateAndTestOne();

lib/Fuzzer/FuzzerLoop.cpp

@@ -175,6 +175,11 @@ void Fuzzer::StaticCrashSignalCallback() {
   F->CrashCallback();
 }
 
+void Fuzzer::StaticExitCallback() {
+  assert(F);
+  F->ExitCallback();
+}
+
 void Fuzzer::StaticInterruptCallback() {
   assert(F);
   F->InterruptCallback();
@@ -198,6 +203,19 @@ void Fuzzer::CrashCallback() {
   _Exit(Options.ErrorExitCode);  // Stop right now.
 }
 
+void Fuzzer::ExitCallback() {
+  if (!RunningCB)
+    return; // This exit did not come from the user callback
+  Printf("==%lu== ERROR: libFuzzer: fuzz target exited\n", GetPid());
+  if (EF->__sanitizer_print_stack_trace)
+    EF->__sanitizer_print_stack_trace();
+  Printf("SUMMARY: libFuzzer: fuzz target exited\n");
+  DumpCurrentUnit("crash-");
+  PrintFinalStats();
+  _Exit(Options.ErrorExitCode);
+}
+
+
 void Fuzzer::InterruptCallback() {
   Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid());
   PrintFinalStats();

lib/Fuzzer/test/AbsNegAndConstant64Test.cpp

@@ -16,6 +16,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   memcpy(&y, Data + sizeof(x), sizeof(y));
   if (llabs(x) < 0 && y == 0xbaddcafedeadbeefULL) {
     printf("BINGO; Found the target, exiting; x = 0x%lx y 0x%lx\n", x, y);
+    fflush(stdout);
     exit(1);
   }
   return 0;

lib/Fuzzer/test/AbsNegAndConstantTest.cpp

@@ -16,6 +16,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   memcpy(&y, Data + sizeof(x), sizeof(y));
   if (abs(x) < 0 && y == 0xbaddcafe) {
     printf("BINGO; Found the target, exiting; x = 0x%x y 0x%x\n", x, y);
+    fflush(stdout);
     exit(1);
   }
   return 0;

lib/Fuzzer/test/BufferOverflowOnInput.cpp

@@ -7,6 +7,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <iostream>
+#include <ostream>
 
 static volatile bool SeedLargeBuffer;
 
@@ -15,7 +16,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   if (Size >= 4)
     SeedLargeBuffer = true;
   if (Size == 3 && SeedLargeBuffer && Data[3]) {
-    std::cout << "Woops, reading Data[3] w/o crashing\n";
+    std::cout << "Woops, reading Data[3] w/o crashing\n" << std::flush;
     exit(1);
   }
   return 0;

lib/Fuzzer/test/CustomCrossOverTest.cpp

@@ -7,6 +7,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <iostream>
+#include <ostream>
 #include <random>
 #include <string.h>
 
@@ -26,7 +27,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   if (Size && Data[0] == 'a') sink--;
 
   if (Str.find(Target) != std::string::npos) {
-    std::cout << "BINGO; Found the target, exiting\n";
+    std::cout << "BINGO; Found the target, exiting\n" << std::flush;
     exit(1);
   }
   return 0;

lib/Fuzzer/test/CustomMutatorTest.cpp

@@ -7,6 +7,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <iostream>
+#include <ostream>
 
 #include "FuzzerInterface.h"
 
@@ -19,7 +20,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     if (Size > 1 && Data[1] == 'i') {
       Sink = 2;
       if (Size > 2 && Data[2] == '!') {
-        std::cout << "BINGO; Found the target, exiting\n";
+        std::cout << "BINGO; Found the target, exiting\n" << std::flush;
         exit(1);
       }
     }

lib/Fuzzer/test/NthRunCrashTest.cpp

@@ -5,12 +5,13 @@
 #include <cstddef>
 #include <cstdint>
 #include <iostream>
+#include <ostream>
 
 static int Counter;
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   if (Counter++ == 1000) {
-    std::cout << "BINGO; Found the target, exiting\n";
+    std::cout << "BINGO; Found the target, exiting\n" << std::flush;
     exit(1);
   }
   return 0;

lib/Fuzzer/test/RepeatedBytesTest.cpp

@@ -7,6 +7,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <iostream>
+#include <ostream>
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   assert(Data);
@@ -21,7 +22,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     MaxA = GT * CurA + (!GT) * MaxA;
   }
   if (MaxA >= 20) {
-    std::cout << "BINGO; Found the target (Max: " << MaxA << "), exiting\n";
+    std::cout << "BINGO; Found the target (Max: " << MaxA << "), exiting\n"
+              << std::flush;
     exit(0);
   }
   return 0;

lib/Fuzzer/test/SimpleDictionaryTest.cpp

@@ -10,6 +10,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <iostream>
+#include <ostream>
 
 static volatile int Zero = 0;
 
@@ -21,7 +22,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     if (Expected[i] + Zero == Data[i])
       Match++;
   if (Match == strlen(Expected)) {
-    std::cout << "BINGO; Found the target, exiting\n";
+    std::cout << "BINGO; Found the target, exiting\n" << std::flush;
     exit(1);
   }
   return 0;

lib/Fuzzer/test/SimpleTest.cpp

@@ -7,6 +7,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <iostream>
+#include <ostream>
 
 static volatile int Sink;
 
@@ -17,7 +18,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     if (Size > 1 && Data[1] == 'i') {
       Sink = 2;
       if (Size > 2 && Data[2] == '!') {
-        std::cout << "BINGO; Found the target, exiting\n";
+        std::cout << "BINGO; Found the target, exiting\n" << std::flush;
         exit(0);
       }
     }

lib/Fuzzer/test/SimpleThreadedTest.cpp

@@ -7,12 +7,13 @@
 #include <cstdint>
 #include <cstring>
 #include <iostream>
+#include <ostream>
 #include <thread>
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   auto C = [&] {
     if (Size >= 2 && Data[0] == 'H') {
-        std::cout << "BINGO; Found the target, exiting\n";
+        std::cout << "BINGO; Found the target, exiting\n" << std::flush;
         abort();
     }
   };

lib/Fuzzer/test/exit-report.test

@@ -0,0 +1,5 @@
+RUN: not LLVMFuzzer-SimpleTest 2>&1 | FileCheck %s
+
+CHECK: ERROR: libFuzzer: fuzz target exited
+CHECK: SUMMARY: libFuzzer: fuzz target exited
+CHECK: Test unit written to

lib/Fuzzer/test/fuzzer-flags.test

@@ -1,18 +1,18 @@
-RUN: LLVMFuzzer-FlagsTest -foo_bar=1 2>&1 | FileCheck %s --check-prefix=FOO_BAR
+RUN: not LLVMFuzzer-FlagsTest -foo_bar=1 2>&1 | FileCheck %s --check-prefix=FOO_BAR
 FOO_BAR: WARNING: unrecognized flag '-foo_bar=1'; use -help=1 to list all flags
 FOO_BAR: BINGO
 
-RUN: LLVMFuzzer-FlagsTest -runs=10 --max_len=100 2>&1 | FileCheck %s --check-prefix=DASH_DASH
+RUN: not LLVMFuzzer-FlagsTest -runs=10 --max_len=100 2>&1 | FileCheck %s --check-prefix=DASH_DASH
 DASH_DASH: WARNING: did you mean '-max_len=100' (single dash)?
 DASH_DASH: INFO: A corpus is not provided, starting from an empty corpus
 
 RUN: LLVMFuzzer-FlagsTest -help=1 2>&1 | FileCheck %s --check-prefix=NO_INTERNAL
 NO_INTERNAL-NOT: internal flag
 
-RUN: LLVMFuzzer-FlagsTest --foo-bar -runs=10 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU
+RUN: not LLVMFuzzer-FlagsTest --foo-bar -runs=10 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU
 PASSTHRU: BINGO --foo-bar --baz -help=1 test
 
 RUN: mkdir -p %t/T0 %t/T1
 RUN: touch %t/T1/empty
-RUN: LLVMFuzzer-FlagsTest --foo-bar -merge=1 %t/T0 %t/T1 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU-MERGE
+RUN: not LLVMFuzzer-FlagsTest --foo-bar -merge=1 %t/T0 %t/T1 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU-MERGE
 PASSTHRU-MERGE: BINGO --foo-bar --baz -help=1 test

lib/Fuzzer/test/fuzzer-printcovpcs.test

@@ -1,4 +1,4 @@
-RUN: LLVMFuzzer-SimpleTest -print_pcs=1 -seed=1 2>&1 | FileCheck %s --check-prefix=PCS
+RUN: not LLVMFuzzer-SimpleTest -print_pcs=1 -seed=1 2>&1 | FileCheck %s --check-prefix=PCS
 PCS-NOT: NEW_PC
 PCS:INITED
 PCS:NEW_PC: {{0x[a-f0-9]+}}

lib/Fuzzer/test/fuzzer.test

@@ -1,10 +1,10 @@
 CHECK: BINGO
 Done1000000: Done 1000000 runs in
 
-RUN: LLVMFuzzer-SimpleTest 2>&1 | FileCheck %s
+RUN: not LLVMFuzzer-SimpleTest 2>&1 | FileCheck %s
 
 # only_ascii mode. Will perform some minimal self-validation.
-RUN: LLVMFuzzer-SimpleTest -only_ascii=1 2>&1
+RUN: not LLVMFuzzer-SimpleTest -only_ascii=1 2>&1
 
 RUN: LLVMFuzzer-SimpleCmpTest -max_total_time=1 -use_cmp=0 2>&1 | FileCheck %s --check-prefix=MaxTotalTime
 MaxTotalTime: Done {{.*}} runs in {{.}} second(s)
@@ -47,7 +47,7 @@ RUN: not LLVMFuzzer-DSOTest 2>&1 | FileCheck %s --check-prefix=DSO
 DSO: INFO: Loaded 3 modules
 DSO: BINGO
 
-RUN: LLVMFuzzer-SimpleTest  -exit_on_src_pos=SimpleTest.cpp:17                 2>&1 | FileCheck %s --check-prefix=EXIT_ON_SRC_POS
+RUN: LLVMFuzzer-SimpleTest  -exit_on_src_pos=SimpleTest.cpp:18                 2>&1 | FileCheck %s --check-prefix=EXIT_ON_SRC_POS
 RUN: LLVMFuzzer-ShrinkControlFlowTest  -exit_on_src_pos=ShrinkControlFlowTest.cpp:23 2>&1 | FileCheck %s --check-prefix=EXIT_ON_SRC_POS
 EXIT_ON_SRC_POS: INFO: found line matching '{{.*}}', exiting.
 

lib/Fuzzer/test/inline-8bit-counters.test

@@ -1,4 +1,4 @@
 REQUIRES: linux
 CHECK: INFO: Loaded 1 modules with {{.*}} inline 8-bit counters
 CHECK: BINGO
-RUN: LLVMFuzzer-SimpleTest-Inline8bitCounters -runs=1000000 -seed=1 2>&1 | FileCheck %s
+RUN: not LLVMFuzzer-SimpleTest-Inline8bitCounters -runs=1000000 -seed=1 2>&1 | FileCheck %s

lib/Fuzzer/test/repeated-bytes.test

@@ -1,2 +1,2 @@
 CHECK: BINGO
-RUN: LLVMFuzzer-RepeatedBytesTest -seed=1 -runs=1000000 2>&1 | FileCheck %s
+RUN: not LLVMFuzzer-RepeatedBytesTest -seed=1 -runs=1000000 2>&1 | FileCheck %s

lib/Fuzzer/test/trace-pc.test

@@ -1,2 +1,2 @@
 CHECK: BINGO
-RUN: LLVMFuzzer-SimpleTest-TracePC -runs=100000 -seed=1 2>&1 | FileCheck %s
+RUN: not LLVMFuzzer-SimpleTest-TracePC -runs=100000 -seed=1 2>&1 | FileCheck %s

lib/Fuzzer/test/ulimit.test

@@ -1,4 +1,4 @@
 REQUIRES: posix
 
 RUN: ulimit -s 1000
-RUN: LLVMFuzzer-SimpleTest
+RUN: not LLVMFuzzer-SimpleTest