diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h index c1be46ddd7b..be8822df3db 100644 --- a/include/llvm/CodeGen/AsmPrinter.h +++ b/include/llvm/CodeGen/AsmPrinter.h @@ -208,6 +208,8 @@ public: SledKind Kind; bool AlwaysInstrument; const class Function *Fn; + + void emit(int, MCStreamer *, const MCSymbol *) const; }; // All the sleds to be emitted. @@ -216,6 +218,9 @@ public: // Helper function to record a given XRay sled. void recordSled(MCSymbol *Sled, const MachineInstr &MI, SledKind Kind); + /// Emit a table with all XRay instrumentation points. + void emitXRayTable(); + //===------------------------------------------------------------------===// // MachineFunctionPass Implementation. //===------------------------------------------------------------------===// diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index de0a4f0befa..5f15ac1d503 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -37,6 +37,8 @@ #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCSection.h" +#include "llvm/MC/MCSectionELF.h" +#include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbolELF.h" #include "llvm/MC/MCValue.h" @@ -2610,6 +2612,61 @@ AsmPrinterHandler::~AsmPrinterHandler() {} void AsmPrinterHandler::markFunctionEnd() {} +// In the binary's "xray_instr_map" section, an array of these function entries +// describes each instrumentation point. When XRay patches your code, the index +// into this table will be given to your handler as a patch point identifier. +void AsmPrinter::XRayFunctionEntry::emit(int Bytes, MCStreamer *Out, + const MCSymbol *CurrentFnSym) const { + Out->EmitSymbolValue(Sled, Bytes); + Out->EmitSymbolValue(CurrentFnSym, Bytes); + auto Kind8 = static_cast(Kind); + Out->EmitBytes(StringRef(reinterpret_cast(&Kind8), 1)); + Out->EmitBytes( + StringRef(reinterpret_cast(&AlwaysInstrument), 1)); + Out->EmitZeros(2 * Bytes - 2); // Pad the previous two entries +} + +void AsmPrinter::emitXRayTable() { + if (Sleds.empty()) + return; + + auto PrevSection = OutStreamer->getCurrentSectionOnly(); + auto Fn = MF->getFunction(); + MCSection *Section = nullptr; + if (MF->getSubtarget().getTargetTriple().isOSBinFormatELF()) { + if (Fn->hasComdat()) { + Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, + ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, + Fn->getComdat()->getName()); + } else { + Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, + ELF::SHF_ALLOC); + } + } else if (MF->getSubtarget().getTargetTriple().isOSBinFormatMachO()) { + Section = OutContext.getMachOSection("__DATA", "xray_instr_map", 0, + SectionKind::getReadOnlyWithRel()); + } else { + llvm_unreachable("Unsupported target"); + } + + // Before we switch over, we force a reference to a label inside the + // xray_instr_map section. Since this function is always called just + // before the function's end, we assume that this is happening after + // the last return instruction. + + auto WordSizeBytes = TM.getPointerSize(); + MCSymbol *Tmp = OutContext.createTempSymbol("xray_synthetic_", true); + OutStreamer->EmitCodeAlignment(16); + OutStreamer->EmitSymbolValue(Tmp, WordSizeBytes, false); + OutStreamer->SwitchSection(Section); + OutStreamer->EmitLabel(Tmp); + for (const auto &Sled : Sleds) + Sled.emit(WordSizeBytes, OutStreamer.get(), CurrentFnSym); + + OutStreamer->SwitchSection(PrevSection); + Sleds.clear(); +} + void AsmPrinter::recordSled(MCSymbol *Sled, const MachineInstr &MI, SledKind Kind) { auto Fn = MI.getParent()->getParent()->getFunction(); diff --git a/lib/Target/AArch64/AArch64AsmPrinter.cpp b/lib/Target/AArch64/AArch64AsmPrinter.cpp index b2d96a32fd3..efc22189378 100644 --- a/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -76,7 +76,6 @@ public: void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI); void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI); - void EmitXRayTable(); void EmitSled(const MachineInstr &MI, SledKind Kind); /// \brief tblgen'erated driver function for lowering simple MI->MC @@ -95,7 +94,7 @@ public: AArch64FI = F.getInfo(); STI = static_cast(&F.getSubtarget()); bool Result = AsmPrinter::runOnMachineFunction(F); - EmitXRayTable(); + emitXRayTable(); return Result; } @@ -150,59 +149,6 @@ void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) EmitSled(MI, SledKind::TAIL_CALL); } -void AArch64AsmPrinter::EmitXRayTable() -{ - //TODO: merge the logic for ELF XRay sleds at a higher level, so to avoid - // code duplication as it is now for x86_64, ARM32 and AArch64. - if (Sleds.empty()) - return; - - auto PrevSection = OutStreamer->getCurrentSectionOnly(); - auto Fn = MF->getFunction(); - MCSection *Section; - - if (STI->isTargetELF()) { - if (Fn->hasComdat()) - Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, - Fn->getComdat()->getName()); - else - Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC); - } else if (STI->isTargetMachO()) { - Section = OutContext.getMachOSection("__DATA", "xray_instr_map", 0, - SectionKind::getReadOnlyWithRel()); - } else { - llvm_unreachable("Unsupported target"); - } - - // Before we switch over, we force a reference to a label inside the - // xray_instr_map section. Since EmitXRayTable() is always called just - // before the function's end, we assume that this is happening after the - // last return instruction. - // - // We then align the reference to 16 byte boundaries, which we determined - // experimentally to be beneficial to avoid causing decoder stalls. - MCSymbol *Tmp = OutContext.createTempSymbol("xray_synthetic_", true); - OutStreamer->EmitCodeAlignment(16); - OutStreamer->EmitSymbolValue(Tmp, 8, false); - OutStreamer->SwitchSection(Section); - OutStreamer->EmitLabel(Tmp); - for (const auto &Sled : Sleds) { - OutStreamer->EmitSymbolValue(Sled.Sled, 8); - OutStreamer->EmitSymbolValue(CurrentFnSym, 8); - auto Kind = static_cast(Sled.Kind); - OutStreamer->EmitBytes( - StringRef(reinterpret_cast(&Kind), 1)); - OutStreamer->EmitBytes( - StringRef(reinterpret_cast(&Sled.AlwaysInstrument), 1)); - OutStreamer->EmitZeros(14); - } - OutStreamer->SwitchSection(PrevSection); - - Sleds.clear(); -} - void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind) { static const int8_t NoopsInSledCount = 7; diff --git a/lib/Target/ARM/ARMAsmPrinter.cpp b/lib/Target/ARM/ARMAsmPrinter.cpp index f20768ab77a..8ec9cb02813 100644 --- a/lib/Target/ARM/ARMAsmPrinter.cpp +++ b/lib/Target/ARM/ARMAsmPrinter.cpp @@ -164,9 +164,6 @@ bool ARMAsmPrinter::runOnMachineFunction(MachineFunction &MF) { // Emit the rest of the function body. EmitFunctionBody(); - // Emit the XRay table for this function. - EmitXRayTable(); - // If we need V4T thumb mode Register Indirect Jump pads, emit them. // These are created per function, rather than per TU, since it's // relatively easy to exceed the thumb branch range within a TU. diff --git a/lib/Target/ARM/ARMAsmPrinter.h b/lib/Target/ARM/ARMAsmPrinter.h index ce0b04d56d9..93fed10eb2d 100644 --- a/lib/Target/ARM/ARMAsmPrinter.h +++ b/lib/Target/ARM/ARMAsmPrinter.h @@ -113,9 +113,6 @@ public: void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI); void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI); void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI); - // Helper function that emits the XRay sleds we've collected for a particular - // function. - void EmitXRayTable(); private: void EmitSled(const MachineInstr &MI, SledKind Kind); diff --git a/lib/Target/ARM/ARMMCInstLower.cpp b/lib/Target/ARM/ARMMCInstLower.cpp index 293a527b09e..07044b9697b 100644 --- a/lib/Target/ARM/ARMMCInstLower.cpp +++ b/lib/Target/ARM/ARMMCInstLower.cpp @@ -22,9 +22,6 @@ #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCContext.h" -#include "llvm/MC/MCSymbolELF.h" -#include "llvm/MC/MCSectionELF.h" -#include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCInstBuilder.h" #include "llvm/MC/MCStreamer.h" using namespace llvm; @@ -226,38 +223,3 @@ void ARMAsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) { EmitSled(MI, SledKind::TAIL_CALL); } - -void ARMAsmPrinter::EmitXRayTable() -{ - if (Sleds.empty()) - return; - - MCSection *Section = nullptr; - if (Subtarget->isTargetELF()) { - Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC | ELF::SHF_GROUP | - ELF::SHF_MERGE, - 0, CurrentFnSym->getName()); - } else if (Subtarget->isTargetMachO()) { - Section = OutContext.getMachOSection("__DATA", "xray_instr_map", 0, - SectionKind::getReadOnlyWithRel()); - } else { - llvm_unreachable("Unsupported target"); - } - - auto PrevSection = OutStreamer->getCurrentSectionOnly(); - OutStreamer->SwitchSection(Section); - for (const auto &Sled : Sleds) { - OutStreamer->EmitSymbolValue(Sled.Sled, 4); - OutStreamer->EmitSymbolValue(CurrentFnSym, 4); - auto Kind = static_cast(Sled.Kind); - OutStreamer->EmitBytes( - StringRef(reinterpret_cast(&Kind), 1)); - OutStreamer->EmitBytes( - StringRef(reinterpret_cast(&Sled.AlwaysInstrument), 1)); - OutStreamer->EmitZeros(6); - } - OutStreamer->SwitchSection(PrevSection); - - Sleds.clear(); -} diff --git a/lib/Target/X86/X86AsmPrinter.cpp b/lib/Target/X86/X86AsmPrinter.cpp index d42e1187ce6..e1825ca1eda 100644 --- a/lib/Target/X86/X86AsmPrinter.cpp +++ b/lib/Target/X86/X86AsmPrinter.cpp @@ -70,7 +70,7 @@ bool X86AsmPrinter::runOnMachineFunction(MachineFunction &MF) { EmitFunctionBody(); // Emit the XRay table for this function. - EmitXRayTable(); + emitXRayTable(); // We didn't modify anything. return false; diff --git a/lib/Target/X86/X86MCInstLower.cpp b/lib/Target/X86/X86MCInstLower.cpp index 2f69df064e7..a38a4b30b77 100644 --- a/lib/Target/X86/X86MCInstLower.cpp +++ b/lib/Target/X86/X86MCInstLower.cpp @@ -1115,56 +1115,6 @@ void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI, X86MCInstLo OutStreamer->EmitInstruction(TC, getSubtargetInfo()); } -void X86AsmPrinter::EmitXRayTable() { - if (Sleds.empty()) - return; - - auto PrevSection = OutStreamer->getCurrentSectionOnly(); - auto Fn = MF->getFunction(); - MCSection *Section = nullptr; - if (Subtarget->isTargetELF()) { - if (Fn->hasComdat()) { - Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, - Fn->getComdat()->getName()); - } else { - Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, - ELF::SHF_ALLOC); - } - } else if (Subtarget->isTargetMachO()) { - Section = OutContext.getMachOSection("__DATA", "xray_instr_map", 0, - SectionKind::getReadOnlyWithRel()); - } else { - llvm_unreachable("Unsupported target"); - } - - // Before we switch over, we force a reference to a label inside the - // xray_instr_map section. Since EmitXRayTable() is always called just - // before the function's end, we assume that this is happening after the - // last return instruction. - // - // We then align the reference to 16 byte boundaries, which we determined - // experimentally to be beneficial to avoid causing decoder stalls. - MCSymbol *Tmp = OutContext.createTempSymbol("xray_synthetic_", true); - OutStreamer->EmitCodeAlignment(16); - OutStreamer->EmitSymbolValue(Tmp, 8, false); - OutStreamer->SwitchSection(Section); - OutStreamer->EmitLabel(Tmp); - for (const auto &Sled : Sleds) { - OutStreamer->EmitSymbolValue(Sled.Sled, 8); - OutStreamer->EmitSymbolValue(CurrentFnSym, 8); - auto Kind = static_cast(Sled.Kind); - OutStreamer->EmitBytes( - StringRef(reinterpret_cast(&Kind), 1)); - OutStreamer->EmitBytes( - StringRef(reinterpret_cast(&Sled.AlwaysInstrument), 1)); - OutStreamer->EmitZeros(14); - } - OutStreamer->SwitchSection(PrevSection); - - Sleds.clear(); -} - // Returns instruction preceding MBBI in MachineFunction. // If MBBI is the first instruction of the first basic block, returns null. static MachineBasicBlock::const_iterator