diff --git a/tools/bugpoint/ExecutionDriver.cpp b/tools/bugpoint/ExecutionDriver.cpp
index 3d4e61058ff..969eaabaf6e 100644
--- a/tools/bugpoint/ExecutionDriver.cpp
+++ b/tools/bugpoint/ExecutionDriver.cpp
@@ -265,8 +265,11 @@ bool BugDriver::diffProgram(const std::string &BytecodeFile,
                                       &ProgramExitedNonzero);
 
   // If we're checking the program exit code, assume anything nonzero is bad.
-  if (CheckProgramExitCode && ProgramExitedNonzero)
+  if (CheckProgramExitCode && ProgramExitedNonzero) {
+    removeFile(Output);
+    if (RemoveBytecode) removeFile(BytecodeFile);
     return true;
+  }
 
   std::string Error;
   bool FilesDifferent = false;