From 1f6ecd860ac4718f38c4453414143b0d768ed55e Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Tue, 25 Oct 2016 02:04:43 +0000 Subject: [PATCH] [libFuzzer] simplify the code for use_cmp, also use the position hint when available, add a test llvm-svn: 285049 --- lib/Fuzzer/FuzzerDictionary.h | 7 ++ lib/Fuzzer/FuzzerLoop.cpp | 5 -- lib/Fuzzer/FuzzerMutate.cpp | 110 +++++++++++++++++++++++++----- lib/Fuzzer/FuzzerMutate.h | 21 +++--- lib/Fuzzer/FuzzerTracePC.cpp | 49 ------------- lib/Fuzzer/FuzzerTracePC.h | 32 ++++----- lib/Fuzzer/test/SimpleCmpTest.cpp | 3 +- lib/Fuzzer/test/SwapCmpTest.cpp | 3 +- lib/Fuzzer/test/swap-cmp.test | 2 + 9 files changed, 130 insertions(+), 102 deletions(-) create mode 100644 lib/Fuzzer/test/swap-cmp.test diff --git a/lib/Fuzzer/FuzzerDictionary.h b/lib/Fuzzer/FuzzerDictionary.h index 821cf9bd001..c009838ced6 100644 --- a/lib/Fuzzer/FuzzerDictionary.h +++ b/lib/Fuzzer/FuzzerDictionary.h @@ -68,6 +68,13 @@ class DictionaryEntry { size_t GetUseCount() const { return UseCount; } size_t GetSuccessCount() const {return SuccessCount; } + void Print(const char *PrintAfter = "\n") { + PrintASCII(W.data(), W.size()); + if (HasPositionHint()) + Printf("@%zd", GetPositionHint()); + Printf("%s", PrintAfter); + } + private: Word W; size_t PositionHint = std::numeric_limits::max(); diff --git a/lib/Fuzzer/FuzzerLoop.cpp b/lib/Fuzzer/FuzzerLoop.cpp index 369dbe5d79c..177702a3f7e 100644 --- a/lib/Fuzzer/FuzzerLoop.cpp +++ b/lib/Fuzzer/FuzzerLoop.cpp @@ -478,9 +478,6 @@ size_t Fuzzer::RunOne(const uint8_t *Data, size_t Size) { Res = 1; } - if (Res && Options.UseCmp) - TPC.ProcessTORC(MD.GetTraceCmpDictionary(), CurrentUnitData, Size); - auto TimeOfUnit = duration_cast(UnitStopTime - UnitStartTime).count(); if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) && @@ -514,8 +511,6 @@ void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { UnitStartTime = system_clock::now(); ResetCounters(); // Reset coverage right before the callback. TPC.ResetMaps(); - if (Options.UseCmp) - TPC.ResetTORC(); if (Options.UseCounters) TPC.ResetGuards(); int Res = CB(DataCopy, Size); diff --git a/lib/Fuzzer/FuzzerMutate.cpp b/lib/Fuzzer/FuzzerMutate.cpp index 7da9256c562..3b5417f76c1 100644 --- a/lib/Fuzzer/FuzzerMutate.cpp +++ b/lib/Fuzzer/FuzzerMutate.cpp @@ -51,8 +51,7 @@ MutationDispatcher::MutationDispatcher(Random &Rand, }); if(Options.UseCmp) DefaultMutators.push_back( - {&MutationDispatcher::Mutate_AddWordFromTraceCmpDictionary, - "TraceCmpDict"}); + {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"}); if (EF->LLVMFuzzerCustomMutator) Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"}); @@ -175,9 +174,96 @@ size_t MutationDispatcher::Mutate_AddWordFromTemporaryAutoDictionary( return AddWordFromDictionary(TempAutoDictionary, Data, Size, MaxSize); } -size_t MutationDispatcher::Mutate_AddWordFromTraceCmpDictionary( +size_t MutationDispatcher::ApplyDictionaryEntry(uint8_t *Data, size_t Size, + size_t MaxSize, + DictionaryEntry &DE) { + const Word &W = DE.GetW(); + bool UsePositionHint = DE.HasPositionHint() && + DE.GetPositionHint() + W.size() < Size && + Rand.RandBool(); + if (Rand.RandBool()) { // Insert W. + if (Size + W.size() > MaxSize) return 0; + size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size + 1); + memmove(Data + Idx + W.size(), Data + Idx, Size - Idx); + memcpy(Data + Idx, W.data(), W.size()); + Size += W.size(); + } else { // Overwrite some bytes with W. + if (W.size() > Size) return 0; + size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size - W.size()); + memcpy(Data + Idx, W.data(), W.size()); + } + return Size; +} + +// Somewhere in the past we have observed a comparison instructions +// with arguments Arg1 Arg2. This function tries to guess a dictionary +// entry that will satisfy that comparison. +// It first tries to find one of the arguments (possibly swapped) in the +// input and if it succeeds it creates a DE with a position hint. +// Otherwise it creates a DE with one of the arguments w/o a position hint. +template +DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP( + T Arg1, T Arg2, const uint8_t *Data, size_t Size) { + ScopedDoingMyOwnMemmem scoped_doing_my_own_memmem; + if (Rand.RandBool()) Arg1 = Bswap(Arg1); + if (Rand.RandBool()) Arg2 = Bswap(Arg2); + bool HandleFirst = Rand.RandBool(); + T ExistingBytes, DesiredBytes; + Word W; + const uint8_t *End = Data + Size; + for (int Arg = 0; Arg < 2; Arg++) { + ExistingBytes = HandleFirst ? Arg1 : Arg2; + DesiredBytes = HandleFirst ? Arg2 : Arg1; + HandleFirst = !HandleFirst; + W.Set(reinterpret_cast(&DesiredBytes), sizeof(T)); + const size_t kMaxNumPositions = 8; + size_t Positions[kMaxNumPositions]; + size_t NumPositions = 0; + for (const uint8_t *Cur = Data; + Cur < End && NumPositions < kMaxNumPositions; Cur++) { + Cur = (uint8_t *)memmem(Cur, End - Cur, &ExistingBytes, sizeof(T)); + if (!Cur) break; + Positions[NumPositions++] = Cur - Data; + } + if (!NumPositions) break; + return DictionaryEntry(W, Positions[Rand(NumPositions)]); + } + DictionaryEntry DE(W); + return DE; +} + +size_t MutationDispatcher::Mutate_AddWordFromTORC( uint8_t *Data, size_t Size, size_t MaxSize) { - return AddWordFromDictionary(TraceCmpDictionary, Data, Size, MaxSize); + Word W; + DictionaryEntry DE; + bool Debug = false; + if (Rand.RandBool()) { + auto X = TPC.TORC8.Get(Rand.Rand()); + DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); + if (X.A > 10000 &&X.B > 10000) Debug = false; + if (Debug) { + Printf("ZZZ %zx %zx\n", X.A, X.B); + DE.Print(); + } + } else { + auto X = TPC.TORC4.Get(Rand.Rand()); + if ((X.A >> 16) == 0 && (X.B >> 16) == 0 && Rand.RandBool()) + DE = MakeDictionaryEntryFromCMP((uint16_t)X.A, (uint16_t)X.B, Data, + Size); + else + DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); + } + Size = ApplyDictionaryEntry(Data, Size, MaxSize, DE); + if (!Size) return 0; + if (Debug) { + Printf("DONE\n"); + } + DictionaryEntry &DERef = + CmpDictionaryEntriesDeque[CmpDictionaryEntriesDequeIdx++ % + kCmpDictionaryEntriesDequeSize]; + DERef = DE; + CurrentDictionaryEntrySequence.push_back(&DERef); + return Size; } size_t MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary( @@ -190,20 +276,8 @@ size_t MutationDispatcher::AddWordFromDictionary(Dictionary &D, uint8_t *Data, if (Size > MaxSize) return 0; if (D.empty()) return 0; DictionaryEntry &DE = D[Rand(D.size())]; - const Word &W = DE.GetW(); - bool UsePositionHint = DE.HasPositionHint() && - DE.GetPositionHint() + W.size() < Size && Rand.RandBool(); - if (Rand.RandBool()) { // Insert W. - if (Size + W.size() > MaxSize) return 0; - size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size + 1); - memmove(Data + Idx + W.size(), Data + Idx, Size - Idx); - memcpy(Data + Idx, W.data(), W.size()); - Size += W.size(); - } else { // Overwrite some bytes with W. - if (W.size() > Size) return 0; - size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size - W.size()); - memcpy(Data + Idx, W.data(), W.size()); - } + Size = ApplyDictionaryEntry(Data, Size, MaxSize, DE); + if (!Size) return 0; DE.IncUseCount(); CurrentDictionaryEntrySequence.push_back(&DE); return Size; diff --git a/lib/Fuzzer/FuzzerMutate.h b/lib/Fuzzer/FuzzerMutate.h index 125185bda9e..d3c0b001246 100644 --- a/lib/Fuzzer/FuzzerMutate.h +++ b/lib/Fuzzer/FuzzerMutate.h @@ -55,9 +55,8 @@ public: size_t Mutate_AddWordFromTemporaryAutoDictionary(uint8_t *Data, size_t Size, size_t MaxSize); - /// Mutates data by adding a word from the trace-cmp dictionary. - size_t Mutate_AddWordFromTraceCmpDictionary(uint8_t *Data, size_t Size, - size_t MaxSize); + /// Mutates data by adding a word from the TORC. + size_t Mutate_AddWordFromTORC(uint8_t *Data, size_t Size, size_t MaxSize); /// Mutates data by adding a word from the persistent automatic dictionary. size_t Mutate_AddWordFromPersistentAutoDictionary(uint8_t *Data, size_t Size, @@ -92,8 +91,6 @@ public: Random &GetRand() { return Rand; } - Dictionary *GetTraceCmpDictionary() { return &TraceCmpDictionary; } - private: struct Mutator { @@ -110,6 +107,12 @@ private: size_t ToSize, size_t MaxToSize); size_t CopyPartOf(const uint8_t *From, size_t FromSize, uint8_t *To, size_t ToSize); + size_t ApplyDictionaryEntry(uint8_t *Data, size_t Size, size_t MaxSize, + DictionaryEntry &DE); + + template + DictionaryEntry MakeDictionaryEntryFromCMP(T Arg1, T Arg2, + const uint8_t *Data, size_t Size); Random &Rand; const FuzzingOptions &Options; @@ -123,11 +126,13 @@ private: // entries that led to successfull discoveries in the past mutations. Dictionary PersistentAutoDictionary; - // Dictionary from tracing CMP instructions. - Dictionary TraceCmpDictionary; - std::vector CurrentMutatorSequence; std::vector CurrentDictionaryEntrySequence; + + static const size_t kCmpDictionaryEntriesDequeSize = 16; + DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; + size_t CmpDictionaryEntriesDequeIdx = 0; + const InputCorpus *Corpus = nullptr; std::vector MutateInPlaceHere; diff --git a/lib/Fuzzer/FuzzerTracePC.cpp b/lib/Fuzzer/FuzzerTracePC.cpp index ddfee218f3b..ed411f1164b 100644 --- a/lib/Fuzzer/FuzzerTracePC.cpp +++ b/lib/Fuzzer/FuzzerTracePC.cpp @@ -272,55 +272,6 @@ void TracePC::HandleCmp(void *PC, T Arg1, T Arg2) { HandleValueProfile(Idx); } -void TracePC::ProcessTORC(Dictionary *Dict, const uint8_t *Data, size_t Size) { - TORCToDict(TORC8, Dict, Data, Size); - TORCToDict(TORC4, Dict, Data, Size); -} - -template -void TracePC::TORCToDict(const TableOfRecentCompares &TORC, - Dictionary *Dict, const uint8_t *Data, size_t Size) { - ScopedDoingMyOwnMemmem scoped_doing_my_own_memmem; - for (size_t i = 0; i < TORC.kSize; i++) { - T A[2] = {TORC.Table[i][0], TORC.Table[i][1]}; - if (!A[0] && !A[1]) continue; - for (int j = 0; j < 2; j++) - TORCToDict(Dict, A[j], A[!j], Data, Size); - } -} - -template -void TracePC::TORCToDict(Dictionary *Dict, T FindInData, T Substitute, - const uint8_t *Data, size_t Size) { - if (FindInData == Substitute) return; - if (sizeof(T) == 4) { - uint16_t HigherBytes = Substitute >> sizeof(T) * 4; - if (HigherBytes == 0 || HigherBytes == 0xffff) - TORCToDict(Dict, static_cast(FindInData), - static_cast(Substitute), Data, Size); - } - const size_t DataSize = sizeof(T); - const uint8_t *End = Data + Size; - int Attempts = 3; - for (int DoSwap = 0; DoSwap <= 1; DoSwap++) { - for (const uint8_t *Cur = Data; Cur < End && Attempts--; Cur++) { - Cur = (uint8_t *)memmem(Cur, End - Cur, &FindInData, DataSize); - if (!Cur) - break; - size_t Pos = Cur - Data; - Word W(reinterpret_cast(&Substitute), sizeof(Substitute)); - DictionaryEntry DE(W, Pos); - // TODO: evict all entries from Dic if it's full. - Dict->push_back(DE); - // Printf("Dict[%zd] TORC%zd %llx => %llx pos %zd\n", Dict->size(), - // sizeof(T), - // (uint64_t)FindInData, (uint64_t)Substitute, Pos); - } - FindInData = Bswap(FindInData); - Substitute = Bswap(Substitute); - } -} - } // namespace fuzzer extern "C" { diff --git a/lib/Fuzzer/FuzzerTracePC.h b/lib/Fuzzer/FuzzerTracePC.h index 632cd541e2b..7be10dae16d 100644 --- a/lib/Fuzzer/FuzzerTracePC.h +++ b/lib/Fuzzer/FuzzerTracePC.h @@ -27,13 +27,18 @@ namespace fuzzer { template struct TableOfRecentCompares { static const size_t kSize = kSizeT; + struct Pair { + T A, B; + }; void Insert(size_t Idx, T Arg1, T Arg2) { Idx = Idx % kSize; - Table[Idx][0] = Arg1; - Table[Idx][1] = Arg2; + Table[Idx].A = Arg1; + Table[Idx].B = Arg2; } - void Clear() { memset(Table, 0, sizeof(Table)); } - T Table[kSize][2]; + + Pair Get(size_t I) { return Table[I % kSize]; } + + Pair Table[kSize]; }; class TracePC { @@ -67,11 +72,6 @@ class TracePC { memset(Counters, 0, sizeof(Counters)); } - void ResetTORC() { - TORC4.Clear(); - TORC8.Clear(); - } - void UpdateFeatureSet(size_t CurrentElementIdx, size_t CurrentElementSize); void PrintFeatureSet(); @@ -88,7 +88,9 @@ class TracePC { bool UsingTracePcGuard() const {return NumModules; } - void ProcessTORC(Dictionary *Dict, const uint8_t *Data, size_t Size); + static const size_t kTORCSize = 1 << 5; + TableOfRecentCompares TORC4; + TableOfRecentCompares TORC8; private: bool UseCounters = false; @@ -113,9 +115,6 @@ private: static const size_t kNumCounters = 1 << 14; alignas(8) uint8_t Counters[kNumCounters]; - static const size_t kTORCSize = 1 << 12; - TableOfRecentCompares TORC4; - TableOfRecentCompares TORC8; void TORCInsert(size_t Idx, uint8_t Arg1, uint8_t Arg2) { // Do nothing, too small to be interesting. } @@ -129,13 +128,6 @@ private: TORC8.Insert(Idx, Arg1, Arg2); } - template - void TORCToDict(const TableOfRecentCompares &TORC, - Dictionary *Dict, const uint8_t *Data, size_t Size); - template - void TORCToDict(Dictionary *Dict, T FindInData, T Substitute, - const uint8_t *Data, size_t Size); - static const size_t kNumPCs = 1 << 24; uintptr_t PCs[kNumPCs]; diff --git a/lib/Fuzzer/test/SimpleCmpTest.cpp b/lib/Fuzzer/test/SimpleCmpTest.cpp index 54dc016ce84..3a2871c6311 100644 --- a/lib/Fuzzer/test/SimpleCmpTest.cpp +++ b/lib/Fuzzer/test/SimpleCmpTest.cpp @@ -25,7 +25,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { z < -10000 && z >= -10005 && z != -10003 && - a == 4242) { + a == 4242 && + true) { fprintf(stderr, "BINGO; Found the target: size %zd (%zd, %zd, %d, %d), exiting.\n", Size, x, y, z, a); exit(1); diff --git a/lib/Fuzzer/test/SwapCmpTest.cpp b/lib/Fuzzer/test/SwapCmpTest.cpp index 3c1199e72fb..f79db4ccf71 100644 --- a/lib/Fuzzer/test/SwapCmpTest.cpp +++ b/lib/Fuzzer/test/SwapCmpTest.cpp @@ -21,8 +21,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { z = __builtin_bswap16(z); if (x == 0x46555A5A5A5A5546ULL && + z == 0x4F4B && y == 0x66757A7A && - z == 0x4F4B + true ) { if (Data[Size - 3] == 'z') { fprintf(stderr, "BINGO; Found the target\n"); diff --git a/lib/Fuzzer/test/swap-cmp.test b/lib/Fuzzer/test/swap-cmp.test new file mode 100644 index 00000000000..d873a321e0f --- /dev/null +++ b/lib/Fuzzer/test/swap-cmp.test @@ -0,0 +1,2 @@ +CHECK: BINGO +RUN: not LLVMFuzzer-SwapCmpTest -seed=1 -use_cmp=1 -runs=10000000 2>&1 | FileCheck %s