diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h index 573ea90cfd0..a6b2850ccd2 100644 --- a/include/llvm/ProfileData/InstrProf.h +++ b/include/llvm/ProfileData/InstrProf.h @@ -598,6 +598,28 @@ struct InstrProfRecord { InstrProfRecord() = default; InstrProfRecord(StringRef Name, uint64_t Hash, std::vector Counts) : Name(Name), Hash(Hash), Counts(std::move(Counts)) {} + InstrProfRecord(InstrProfRecord &&) = default; + InstrProfRecord(const InstrProfRecord &RHS) + : Name(RHS.Name), Hash(RHS.Hash), Counts(RHS.Counts), SIPE(RHS.SIPE), + ValueData(RHS.ValueData + ? llvm::make_unique(*RHS.ValueData) + : nullptr) {} + InstrProfRecord &operator=(InstrProfRecord &&) = default; + InstrProfRecord &operator=(const InstrProfRecord &RHS) { + Name = RHS.Name; + Hash = RHS.Hash; + Counts = RHS.Counts; + SIPE = RHS.SIPE; + if (!RHS.ValueData) { + ValueData = nullptr; + return *this; + } + if (!ValueData) + ValueData = llvm::make_unique(*RHS.ValueData); + else + *ValueData = *RHS.ValueData; + return *this; + } using ValueMapType = std::vector>; @@ -647,12 +669,9 @@ struct InstrProfRecord { /// Sort value profile data (per site) by count. void sortValueData() { - for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) { - std::vector &SiteRecords = - getValueSitesForKind(Kind); - for (auto &SR : SiteRecords) + for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) + for (auto &SR : getValueSitesForKind(Kind)) SR.sortByCount(); - } } /// Clear value data entries and edge counters. @@ -662,36 +681,54 @@ struct InstrProfRecord { } /// Clear value data entries - void clearValueData() { - for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) - getValueSitesForKind(Kind).clear(); - } + void clearValueData() { ValueData = nullptr; } /// Get the error contained within the record's soft error counter. Error takeError() { return SIPE.takeError(); } private: - std::vector IndirectCallSites; - std::vector MemOPSizes; + struct ValueProfData { + std::vector IndirectCallSites; + std::vector MemOPSizes; + }; + std::unique_ptr ValueData; - const std::vector & + MutableArrayRef + getValueSitesForKind(uint32_t ValueKind) { + // Cast to /add/ const (should be an implicit_cast, ideally, if that's ever + // implemented in LLVM) to call the const overload of this function, then + // cast away the constness from the result. + auto AR = const_cast(this)->getValueSitesForKind( + ValueKind); + return makeMutableArrayRef( + const_cast(AR.data()), AR.size()); + } + ArrayRef getValueSitesForKind(uint32_t ValueKind) const { + if (!ValueData) + return None; switch (ValueKind) { case IPVK_IndirectCallTarget: - return IndirectCallSites; + return ValueData->IndirectCallSites; case IPVK_MemOPSize: - return MemOPSizes; + return ValueData->MemOPSizes; default: llvm_unreachable("Unknown value kind!"); } - return IndirectCallSites; } std::vector & - getValueSitesForKind(uint32_t ValueKind) { - return const_cast &>( - const_cast(this) - ->getValueSitesForKind(ValueKind)); + getOrCreateValueSitesForKind(uint32_t ValueKind) { + if (!ValueData) + ValueData = llvm::make_unique(); + switch (ValueKind) { + case IPVK_IndirectCallTarget: + return ValueData->IndirectCallSites; + case IPVK_MemOPSize: + return ValueData->MemOPSizes; + default: + llvm_unreachable("Unknown value kind!"); + } } // Map indirect call target name hash to name string. @@ -765,9 +802,9 @@ uint64_t InstrProfRecord::getValueForSite(InstrProfValueData Dest[], } void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) { - std::vector &ValueSites = - getValueSitesForKind(ValueKind); - ValueSites.reserve(NumValueSites); + if (!NumValueSites) + return; + getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites); } inline support::endianness getHostEndianness() { diff --git a/lib/ProfileData/InstrProf.cpp b/lib/ProfileData/InstrProf.cpp index 1dc596fa7ed..a1d18724fcd 100644 --- a/lib/ProfileData/InstrProf.cpp +++ b/lib/ProfileData/InstrProf.cpp @@ -504,9 +504,11 @@ void InstrProfRecord::mergeValueProfData(uint32_t ValueKind, SIPE.addError(instrprof_error::value_site_count_mismatch); return; } + if (!ThisNumValueSites) + return; std::vector &ThisSiteRecords = - getValueSitesForKind(ValueKind); - std::vector &OtherSiteRecords = + getOrCreateValueSitesForKind(ValueKind); + MutableArrayRef OtherSiteRecords = Src.getValueSitesForKind(ValueKind); for (uint32_t I = 0; I < ThisNumValueSites; I++) ThisSiteRecords[I].merge(SIPE, OtherSiteRecords[I], Weight); @@ -533,11 +535,8 @@ void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight) { } void InstrProfRecord::scaleValueProfData(uint32_t ValueKind, uint64_t Weight) { - uint32_t ThisNumValueSites = getNumValueSites(ValueKind); - std::vector &ThisSiteRecords = - getValueSitesForKind(ValueKind); - for (uint32_t I = 0; I < ThisNumValueSites; I++) - ThisSiteRecords[I].scale(SIPE, Weight); + for (auto &R : getValueSitesForKind(ValueKind)) + R.scale(SIPE, Weight); } void InstrProfRecord::scale(uint64_t Weight) { @@ -583,7 +582,7 @@ void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site, VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap); } std::vector &ValueSites = - getValueSitesForKind(ValueKind); + getOrCreateValueSitesForKind(ValueKind); if (N == 0) ValueSites.emplace_back(); else