mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2024-11-25 20:23:11 +01:00
3594bb4c1b
Sample re-annotation is required in LTO time to achieve a reasonable post-inline profile quality. However, we have seen that such LTO-time re-annotation degrades profile quality. This is mainly caused by preLTO code duplication that is done by passes such as loop unrolling, jump threading, indirect call promotion etc, where samples corresponding to a source location are aggregated multiple times due to the duplicates. In this change we are introducing a concept of distribution factor for pseudo probes so that samples can be distributed for duplicated probes scaled by a factor. We hope that optimizations duplicating code well-maintain the branch frequency information (BFI) based on which probe distribution factors are calculated. Distribution factors are updated at the end of preLTO pipeline to reflect an estimated portion of the real execution count. This change also introduces a pseudo probe verifier that can be run after each IR passes to detect duplicated pseudo probes. A saturated distribution factor stands for 1.0. A pesudo probe will carry a factor with the value ranged from 0.0 to 1.0. A 64-bit integral distribution factor field that represents [0.0, 1.0] is associated to each block probe. Unfortunately this cannot be done for callsite probes due to the size limitation of a 32-bit Dwarf discriminator. A 7-bit distribution factor is used instead. Changes are also needed to the sample profile inliner to deal with prorated callsite counts. Call sites duplicated by PreLTO passes, when later on inlined in LTO time, should have the callees’s probe prorated based on the Prelink-computed distribution factors. The distribution factors should also be taken into account when computing hotness for inline candidates. Also, Indirect call promotion results in multiple callisites. The original samples should be distributed across them. This is fixed by adjusting the callisites' distribution factors. Reviewed By: wmi Differential Revision: https://reviews.llvm.org/D93264
100 lines
3.7 KiB
C++
100 lines
3.7 KiB
C++
//===- PseudoProbe.cpp - Pseudo Probe Helpers -----------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file implements the helpers to manipulate pseudo probe IR intrinsic
|
|
// calls.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/IR/PseudoProbe.h"
|
|
#include "llvm/IR/DebugInfoMetadata.h"
|
|
#include "llvm/IR/IRBuilder.h"
|
|
#include "llvm/IR/Instruction.h"
|
|
|
|
using namespace llvm;
|
|
|
|
namespace llvm {
|
|
|
|
Optional<PseudoProbe> extractProbeFromDiscriminator(const Instruction &Inst) {
|
|
assert(isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst) &&
|
|
"Only call instructions should have pseudo probe encodes as their "
|
|
"Dwarf discriminators");
|
|
if (const DebugLoc &DLoc = Inst.getDebugLoc()) {
|
|
const DILocation *DIL = DLoc;
|
|
auto Discriminator = DIL->getDiscriminator();
|
|
if (DILocation::isPseudoProbeDiscriminator(Discriminator)) {
|
|
PseudoProbe Probe;
|
|
Probe.Id =
|
|
PseudoProbeDwarfDiscriminator::extractProbeIndex(Discriminator);
|
|
Probe.Type =
|
|
PseudoProbeDwarfDiscriminator::extractProbeType(Discriminator);
|
|
Probe.Attr =
|
|
PseudoProbeDwarfDiscriminator::extractProbeAttributes(Discriminator);
|
|
Probe.Factor =
|
|
PseudoProbeDwarfDiscriminator::extractProbeFactor(Discriminator) /
|
|
(float)PseudoProbeDwarfDiscriminator::FullDistributionFactor;
|
|
return Probe;
|
|
}
|
|
}
|
|
return None;
|
|
}
|
|
|
|
Optional<PseudoProbe> extractProbe(const Instruction &Inst) {
|
|
if (const auto *II = dyn_cast<PseudoProbeInst>(&Inst)) {
|
|
PseudoProbe Probe;
|
|
Probe.Id = II->getIndex()->getZExtValue();
|
|
Probe.Type = (uint32_t)PseudoProbeType::Block;
|
|
Probe.Attr = II->getAttributes()->getZExtValue();
|
|
Probe.Factor = II->getFactor()->getZExtValue() /
|
|
(float)PseudoProbeFullDistributionFactor;
|
|
return Probe;
|
|
}
|
|
|
|
if (isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst))
|
|
return extractProbeFromDiscriminator(Inst);
|
|
|
|
return None;
|
|
}
|
|
|
|
void setProbeDistributionFactor(Instruction &Inst, float Factor) {
|
|
assert(Factor >= 0 && Factor <= 1 &&
|
|
"Distribution factor must be in [0, 1.0]");
|
|
if (auto *II = dyn_cast<PseudoProbeInst>(&Inst)) {
|
|
IRBuilder<> Builder(&Inst);
|
|
uint64_t IntFactor = PseudoProbeFullDistributionFactor;
|
|
if (Factor < 1)
|
|
IntFactor *= Factor;
|
|
auto OrigFactor = II->getFactor()->getZExtValue();
|
|
if (IntFactor != OrigFactor)
|
|
II->replaceUsesOfWith(II->getFactor(), Builder.getInt64(IntFactor));
|
|
} else if (isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst)) {
|
|
if (const DebugLoc &DLoc = Inst.getDebugLoc()) {
|
|
const DILocation *DIL = DLoc;
|
|
auto Discriminator = DIL->getDiscriminator();
|
|
if (DILocation::isPseudoProbeDiscriminator(Discriminator)) {
|
|
auto Index =
|
|
PseudoProbeDwarfDiscriminator::extractProbeIndex(Discriminator);
|
|
auto Type =
|
|
PseudoProbeDwarfDiscriminator::extractProbeType(Discriminator);
|
|
auto Attr = PseudoProbeDwarfDiscriminator::extractProbeAttributes(
|
|
Discriminator);
|
|
// Round small factors to 0 to avoid over-counting.
|
|
uint32_t IntFactor =
|
|
PseudoProbeDwarfDiscriminator::FullDistributionFactor;
|
|
if (Factor < 1)
|
|
IntFactor *= Factor;
|
|
uint32_t V = PseudoProbeDwarfDiscriminator::packProbeData(
|
|
Index, Type, Attr, IntFactor);
|
|
DIL = DIL->cloneWithDiscriminator(V);
|
|
Inst.setDebugLoc(DIL);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} // namespace llvm
|