From 388284330d709c36a87c500fccd8a3c067f450e2 Mon Sep 17 00:00:00 2001 From: Teresa Johnson Date: Mon, 26 Nov 2018 20:40:37 +0000 Subject: [PATCH] [ThinLTO] Consolidate cache key computation between new/old LTO APIs Summary: The old legacy LTO API had a separate cache key computation, which was a subset of the cache key computation in the new LTO API (from what I can tell this is largely just because certain features such as CFI, dsoLocal, etc are only utilized via the new LTO API). However, having separate computations is unnecessary (much of the code is duplicated), and can lead to bugs when adding new optimizations if both cache computation algorithms aren't updated properly - it's much easier to maintain if we have a single facility. This patch refactors the old LTO API code to use the cache key computation from the new LTO API. To do this, we set up an lto::Config object and fill in the fields that the old LTO was hashing (the others will just use the defaults). There are two notable changes: - I added a Freestanding flag to the LTO Config. Currently this is only used by the legacy LTO API. In the patch that added it (D30791) I had asked about adding it to the new LTO API, but it looks like that was not addressed. This should probably be discussed as a follow up to this change, as it is orthogonal. - The legacy LTO API had some code that was hashing the GUID of all preserved symbols defined in the module. I looked back at the history of this (which was added with the original hashing in the legacy LTO API in D18494), and there is a comment in the review thread that it was added in preparation for future internalization. We now do the internalization of course, and that is handled in the new LTO API cache key computation by hashing the recorded linkage type of all defined globals. Therefore I didn't try to move over and keep the preserved symbols handling. Reviewers: steven_wu, pcc Subscribers: mehdi_amini, inglorion, eraman, dexonsmith, dang, llvm-commits Differential Revision: https://reviews.llvm.org/D54635 llvm-svn: 347592 --- include/llvm/LTO/Config.h | 4 ++ include/llvm/LTO/LTO.h | 13 ++++ lib/LTO/LTO.cpp | 11 ++-- lib/LTO/ThinLTOCodeGenerator.cpp | 104 +++++-------------------------- 4 files changed, 40 insertions(+), 92 deletions(-) diff --git a/include/llvm/LTO/Config.h b/include/llvm/LTO/Config.h index c0ad32f485c..7058602c3ee 100644 --- a/include/llvm/LTO/Config.h +++ b/include/llvm/LTO/Config.h @@ -49,6 +49,10 @@ struct Config { /// Use the new pass manager bool UseNewPM = false; + /// Flag to indicate that the optimizer should not assume builtins are present + /// on the target. + bool Freestanding = false; + /// Disable entirely the optimizer, including importing for ThinLTO bool CodeGenOnly = false; diff --git a/include/llvm/LTO/LTO.h b/include/llvm/LTO/LTO.h index 2a3bc9181af..15390873056 100644 --- a/include/llvm/LTO/LTO.h +++ b/include/llvm/LTO/LTO.h @@ -60,6 +60,19 @@ void thinLTOInternalizeAndPromoteInIndex( ModuleSummaryIndex &Index, function_ref isExported); +/// Computes a unique hash for the Module considering the current list of +/// export/import and other global analysis results. +/// The hash is produced in \p Key. +void computeLTOCacheKey( + SmallString<40> &Key, const lto::Config &Conf, + const ModuleSummaryIndex &Index, StringRef ModuleID, + const FunctionImporter::ImportMapTy &ImportList, + const FunctionImporter::ExportSetTy &ExportList, + const std::map &ResolvedODR, + const GVSummaryMapTy &DefinedGlobals, + const std::set &CfiFunctionDefs = {}, + const std::set &CfiFunctionDecls = {}); + namespace lto { /// Given the original \p Path to an output file, replace any path diff --git a/lib/LTO/LTO.cpp b/lib/LTO/LTO.cpp index 68fd93a5d9c..f4840db9fe0 100644 --- a/lib/LTO/LTO.cpp +++ b/lib/LTO/LTO.cpp @@ -61,10 +61,10 @@ cl::opt EnableLTOInternalization( "enable-lto-internalization", cl::init(true), cl::Hidden, cl::desc("Enable global value internalization in LTO")); -// Returns a unique hash for the Module considering the current list of +// Computes a unique hash for the Module considering the current list of // export/import and other global analysis results. // The hash is produced in \p Key. -static void computeCacheKey( +void llvm::computeLTOCacheKey( SmallString<40> &Key, const Config &Conf, const ModuleSummaryIndex &Index, StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, @@ -132,6 +132,7 @@ static void computeCacheKey( AddUnsigned(Conf.CGFileType); AddUnsigned(Conf.OptLevel); AddUnsigned(Conf.UseNewPM); + AddUnsigned(Conf.Freestanding); AddString(Conf.OptPipeline); AddString(Conf.AAPipeline); AddString(Conf.OverrideTriple); @@ -983,9 +984,9 @@ public: SmallString<40> Key; // The module may be cached, this helps handling it. - computeCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList, ExportList, - ResolvedODR, DefinedGlobals, CfiFunctionDefs, - CfiFunctionDecls); + computeLTOCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList, + ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, + CfiFunctionDecls); if (AddStreamFn CacheAddStream = Cache(Task, Key)) return RunThinBackend(CacheAddStream); diff --git a/lib/LTO/ThinLTOCodeGenerator.cpp b/lib/LTO/ThinLTOCodeGenerator.cpp index 0cb47f2b34b..e9633b015c1 100644 --- a/lib/LTO/ThinLTOCodeGenerator.cpp +++ b/lib/LTO/ThinLTOCodeGenerator.cpp @@ -298,8 +298,7 @@ public: const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, const std::map &ResolvedODR, - const GVSummaryMapTy &DefinedGVSummaries, - const DenseSet &PreservedSymbols, unsigned OptLevel, + const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel, bool Freestanding, const TargetMachineBuilder &TMBuilder) { if (CachePath.empty()) return; @@ -308,95 +307,26 @@ public: // The module does not have an entry, it can't have a hash at all return; - // Compute the unique hash for this entry - // This is based on the current compiler version, the module itself, the - // export list, the hash for every single module in the import list, the - // list of ResolvedODR for the module, and the list of preserved symbols. - - // Include the hash for the current module - auto ModHash = Index.getModuleHash(ModuleID); - - if (all_of(ModHash, [](uint32_t V) { return V == 0; })) + if (all_of(Index.getModuleHash(ModuleID), + [](uint32_t V) { return V == 0; })) // No hash entry, no caching! return; - SHA1 Hasher; - - // Include the parts of the LTO configuration that affect code generation. - auto AddString = [&](StringRef Str) { - Hasher.update(Str); - Hasher.update(ArrayRef{0}); - }; - auto AddUnsigned = [&](unsigned I) { - uint8_t Data[4]; - Data[0] = I; - Data[1] = I >> 8; - Data[2] = I >> 16; - Data[3] = I >> 24; - Hasher.update(ArrayRef{Data, 4}); - }; - - // Start with the compiler revision - Hasher.update(LLVM_VERSION_STRING); -#ifdef LLVM_REVISION - Hasher.update(LLVM_REVISION); -#endif - - // Hash the optimization level and the target machine settings. - AddString(TMBuilder.MCpu); - // FIXME: Hash more of Options. For now all clients initialize Options from - // command-line flags (which is unsupported in production), but may set - // RelaxELFRelocations. The clang driver can also pass FunctionSections, - // DataSections and DebuggerTuning via command line flags. - AddUnsigned(TMBuilder.Options.RelaxELFRelocations); - AddUnsigned(TMBuilder.Options.FunctionSections); - AddUnsigned(TMBuilder.Options.DataSections); - AddUnsigned((unsigned)TMBuilder.Options.DebuggerTuning); - AddString(TMBuilder.MAttr); - if (TMBuilder.RelocModel) - AddUnsigned(*TMBuilder.RelocModel); - AddUnsigned(TMBuilder.CGOptLevel); - AddUnsigned(OptLevel); - AddUnsigned(Freestanding); - - Hasher.update(ArrayRef((uint8_t *)&ModHash[0], sizeof(ModHash))); - for (auto F : ExportList) - // The export list can impact the internalization, be conservative here - Hasher.update(ArrayRef((uint8_t *)&F, sizeof(F))); - - // Include the hash for every module we import functions from - for (auto &Entry : ImportList) { - auto ModHash = Index.getModuleHash(Entry.first()); - Hasher.update(ArrayRef((uint8_t *)&ModHash[0], sizeof(ModHash))); - for (auto Guid : Entry.second) - if (auto *GVS = dyn_cast( - Index.getGlobalValueSummary(Guid, false))) - AddUnsigned(GVS->isReadOnly()); - } - - // Include the hash for the resolved ODR. - for (auto &Entry : ResolvedODR) { - Hasher.update(ArrayRef((const uint8_t *)&Entry.first, - sizeof(GlobalValue::GUID))); - Hasher.update(ArrayRef((const uint8_t *)&Entry.second, - sizeof(GlobalValue::LinkageTypes))); - } - - // Include the hash for the preserved symbols. - for (auto &Entry : PreservedSymbols) { - if (DefinedGVSummaries.count(Entry)) - Hasher.update( - ArrayRef((const uint8_t *)&Entry, sizeof(GlobalValue::GUID))); - } - - for (auto &Entry : DefinedGVSummaries) - if (auto *GVS = dyn_cast(Entry.second)) - AddUnsigned(GVS->isReadOnly()); + llvm::lto::Config Conf; + Conf.OptLevel = OptLevel; + Conf.Options = TMBuilder.Options; + Conf.CPU = TMBuilder.MCpu; + Conf.MAttrs.push_back(TMBuilder.MAttr); + Conf.RelocModel = TMBuilder.RelocModel; + Conf.CGOptLevel = TMBuilder.CGOptLevel; + Conf.Freestanding = Freestanding; + SmallString<40> Key; + computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList, + ResolvedODR, DefinedGVSummaries); // This choice of file name allows the cache to be pruned (see pruneCache() // in include/llvm/Support/CachePruning.h). - sys::path::append(EntryPath, CachePath, - "llvmcache-" + toHex(Hasher.result())); + sys::path::append(EntryPath, CachePath, "llvmcache-" + Key); } // Access the path to this entry in the cache. @@ -998,8 +928,8 @@ void ThinLTOCodeGenerator::run() { ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier, ImportLists[ModuleIdentifier], ExportList, ResolvedODR[ModuleIdentifier], - DefinedGVSummaries, GUIDPreservedSymbols, - OptLevel, Freestanding, TMBuilder); + DefinedGVSummaries, OptLevel, Freestanding, + TMBuilder); auto CacheEntryPath = CacheEntry.getEntryPath(); {