diff --git a/tools/llvm-mca/ExecuteStage.cpp b/tools/llvm-mca/ExecuteStage.cpp index eb71ab37084..7f604d66ff5 100644 --- a/tools/llvm-mca/ExecuteStage.cpp +++ b/tools/llvm-mca/ExecuteStage.cpp @@ -52,57 +52,31 @@ bool ExecuteStage::isAvailable(const InstRef &IR) const { return true; } -void ExecuteStage::reclaimSchedulerResources() { - SmallVector ResourcesFreed; - HWS.reclaimSimulatedResources(ResourcesFreed); - for (const ResourceRef &RR : ResourcesFreed) - notifyResourceAvailable(RR); -} +Error ExecuteStage::issueInstruction(InstRef &IR) { + SmallVector, 4> Used; + SmallVector Ready; + HWS.issueInstruction(IR, Used, Ready); -Error ExecuteStage::updateSchedulerQueues() { - SmallVector InstructionIDs; - HWS.updateIssuedSet(InstructionIDs); - for (InstRef &IR : InstructionIDs) { + const InstrDesc &Desc = IR.getInstruction()->getDesc(); + notifyReleasedBuffers(Desc.Buffers); + notifyInstructionIssued(IR, Used); + if (IR.getInstruction()->isExecuted()) { notifyInstructionExecuted(IR); //FIXME: add a buffer of executed instructions. if (Error S = moveToTheNextStage(IR)) return S; } - InstructionIDs.clear(); - HWS.updatePendingQueue(InstructionIDs); - for (const InstRef &IR : InstructionIDs) - notifyInstructionReady(IR); + for (const InstRef &I : Ready) + notifyInstructionReady(I); return ErrorSuccess(); } Error ExecuteStage::issueReadyInstructions() { - SmallVector InstructionIDs; InstRef IR = HWS.select(); while (IR.isValid()) { - SmallVector, 4> Used; - HWS.issueInstruction(IR, Used); - - // Reclaim instruction resources and perform notifications. - const InstrDesc &Desc = IR.getInstruction()->getDesc(); - notifyReleasedBuffers(Desc.Buffers); - notifyInstructionIssued(IR, Used); - if (IR.getInstruction()->isExecuted()) { - notifyInstructionExecuted(IR); - //FIXME: add a buffer of executed instructions. - if (Error S = moveToTheNextStage(IR)) - return S; - } - - // Instructions that have been issued during this cycle might have unblocked - // other dependent instructions. Dependent instructions may be issued during - // this same cycle if operands have ReadAdvance entries. Promote those - // instructions to the ReadySet and tell to the caller that we need - // another round of 'issue()'. - HWS.promoteToReadySet(InstructionIDs); - for (const InstRef &I : InstructionIDs) - notifyInstructionReady(I); - InstructionIDs.clear(); + if (Error Err = issueInstruction(IR)) + return Err; // Select the next instruction to issue. IR = HWS.select(); @@ -111,22 +85,26 @@ Error ExecuteStage::issueReadyInstructions() { return ErrorSuccess(); } -// The following routine is the maintenance routine of the ExecuteStage. -// It is responsible for updating the hardware scheduler (HWS), including -// reclaiming the HWS's simulated hardware resources, as well as updating the -// HWS's queues. -// -// This routine also processes the instructions that are ready for issuance. -// These instructions are managed by the HWS's ready queue and can be accessed -// via the Scheduler::select() routine. -// -// Notifications are issued to this stage's listeners when instructions are -// moved between the HWS's queues. In particular, when an instruction becomes -// ready or executed. Error ExecuteStage::cycleStart() { - reclaimSchedulerResources(); - if (Error S = updateSchedulerQueues()) - return S; + llvm::SmallVector Freed; + llvm::SmallVector Executed; + llvm::SmallVector Ready; + + HWS.cycleEvent(Freed, Executed, Ready); + + for (const ResourceRef &RR : Freed) + notifyResourceAvailable(RR); + + for (InstRef &IR : Executed) { + notifyInstructionExecuted(IR); + //FIXME: add a buffer of executed instructions. + if (Error S = moveToTheNextStage(IR)) + return S; + } + + for (const InstRef &IR : Ready) + notifyInstructionReady(IR); + return issueReadyInstructions(); } @@ -157,18 +135,7 @@ Error ExecuteStage::execute(InstRef &IR) { return ErrorSuccess(); // Issue IR to the underlying pipelines. - SmallVector, 4> Used; - HWS.issueInstruction(IR, Used); - - // Perform notifications. - notifyReleasedBuffers(Desc.Buffers); - notifyInstructionIssued(IR, Used); - if (IR.getInstruction()->isExecuted()) { - notifyInstructionExecuted(IR); - //FIXME: add a buffer of executed instructions. - return moveToTheNextStage(IR); - } - return ErrorSuccess(); + return issueInstruction(IR); } void ExecuteStage::notifyInstructionExecuted(const InstRef &IR) { diff --git a/tools/llvm-mca/ExecuteStage.h b/tools/llvm-mca/ExecuteStage.h index 85938d6ac51..13ca612023a 100644 --- a/tools/llvm-mca/ExecuteStage.h +++ b/tools/llvm-mca/ExecuteStage.h @@ -28,9 +28,10 @@ namespace mca { class ExecuteStage final : public Stage { Scheduler &HWS; - // The following routines are used to maintain the HWS. - void reclaimSchedulerResources(); - llvm::Error updateSchedulerQueues(); + llvm::Error issueInstruction(InstRef &IR); + + // Called at the beginning of each cycle to issue already dispatched + // instructions to the underlying pipelines. llvm::Error issueReadyInstructions(); ExecuteStage(const ExecuteStage &Other) = delete; @@ -47,6 +48,14 @@ public: // are still instructions in-flight in the out-of-order backend. bool hasWorkToComplete() const override { return false; } bool isAvailable(const InstRef &IR) const override; + + // Notifies the scheduler that a new cycle just started. + // + // This method notifies the scheduler that a new cycle started. + // This method is also responsible for notifying listeners about instructions + // state changes, and processor resources freed by the scheduler. + // Instructions that transitioned to the 'Executed' state are automatically + // moved to the next stage (i.e. RetireStage). llvm::Error cycleStart() override; llvm::Error execute(InstRef &IR) override; diff --git a/tools/llvm-mca/Instruction.h b/tools/llvm-mca/Instruction.h index 55dc644a587..33e867fedce 100644 --- a/tools/llvm-mca/Instruction.h +++ b/tools/llvm-mca/Instruction.h @@ -107,6 +107,9 @@ class WriteState { // that we don't break the WAW, and the two writes can be merged together. const WriteState *DependentWrite; + // Number of writes that are in a WAW dependency with this write. + unsigned NumWriteUsers; + // A list of dependent reads. Users is a set of dependent // reads. A dependent read is added to the set only if CyclesLeft // is "unknown". As soon as CyclesLeft is 'known', each user in the set @@ -119,7 +122,8 @@ public: WriteState(const WriteDescriptor &Desc, unsigned RegID, bool clearsSuperRegs = false) : WD(Desc), CyclesLeft(UNKNOWN_CYCLES), RegisterID(RegID), - ClearsSuperRegs(clearsSuperRegs), DependentWrite(nullptr) {} + ClearsSuperRegs(clearsSuperRegs), DependentWrite(nullptr), + NumWriteUsers(0U) {} WriteState(const WriteState &Other) = delete; WriteState &operator=(const WriteState &Other) = delete; @@ -129,11 +133,15 @@ public: unsigned getLatency() const { return WD.Latency; } void addUser(ReadState *Use, int ReadAdvance); - unsigned getNumUsers() const { return Users.size(); } + + unsigned getNumUsers() const { return Users.size() + NumWriteUsers; } bool clearsSuperRegisters() const { return ClearsSuperRegs; } const WriteState *getDependentWrite() const { return DependentWrite; } - void setDependentWrite(const WriteState *Write) { DependentWrite = Write; } + void setDependentWrite(WriteState *Other) { + DependentWrite = Other; + ++Other->NumWriteUsers; + } // On every cycle, update CyclesLeft and notify dependent users. void cycleEvent(); @@ -330,6 +338,11 @@ public: unsigned getRCUTokenID() const { return RCUTokenID; } int getCyclesLeft() const { return CyclesLeft; } + bool hasDependentUsers() const { + return llvm::any_of( + Defs, [](const UniqueDef &Def) { return Def->getNumUsers() > 0; }); + } + bool isDependencyBreaking() const { return IsDepBreaking; } void setDependencyBreaking() { IsDepBreaking = true; } diff --git a/tools/llvm-mca/RegisterFile.cpp b/tools/llvm-mca/RegisterFile.cpp index ea27a8b9b68..cba17a6a6a1 100644 --- a/tools/llvm-mca/RegisterFile.cpp +++ b/tools/llvm-mca/RegisterFile.cpp @@ -166,7 +166,7 @@ void RegisterFile::addRegisterWrite(WriteRef Write, const RegisterRenamingInfo &RRI = RegisterMappings[RegID].second; if (RRI.RenameAs && RRI.RenameAs != RegID) { RegID = RRI.RenameAs; - const WriteRef &OtherWrite = RegisterMappings[RegID].first; + WriteRef &OtherWrite = RegisterMappings[RegID].first; if (!WS.clearsSuperRegisters()) { // The processor keeps the definition of `RegID` together with register @@ -174,7 +174,8 @@ void RegisterFile::addRegisterWrite(WriteRef Write, // register is allocated. ShouldAllocatePhysRegs = false; - if (OtherWrite.getSourceIndex() != Write.getSourceIndex()) { + if (OtherWrite.getWriteState() && + (OtherWrite.getSourceIndex() != Write.getSourceIndex())) { // This partial write has a false dependency on RenameAs. WS.setDependentWrite(OtherWrite.getWriteState()); } diff --git a/tools/llvm-mca/Scheduler.cpp b/tools/llvm-mca/Scheduler.cpp index 3684ad59bc3..2e0ac378d9c 100644 --- a/tools/llvm-mca/Scheduler.cpp +++ b/tools/llvm-mca/Scheduler.cpp @@ -305,10 +305,19 @@ void Scheduler::issueInstructionImpl( // Release the buffered resources and issue the instruction. void Scheduler::issueInstruction( InstRef &IR, - SmallVectorImpl> &UsedResources) { - const InstrDesc &Desc = IR.getInstruction()->getDesc(); - Resources->releaseBuffers(Desc.Buffers); + SmallVectorImpl> &UsedResources, + SmallVectorImpl &ReadyInstructions) { + const Instruction &Inst = *IR.getInstruction(); + bool HasDependentUsers = Inst.hasDependentUsers(); + + Resources->releaseBuffers(Inst.getDesc().Buffers); issueInstructionImpl(IR, UsedResources); + // Instructions that have been issued during this cycle might have unblocked + // other dependent instructions. Dependent instructions may be issued during + // this same cycle if operands have ReadAdvance entries. Promote those + // instructions to the ReadySet and notify the caller that those are ready. + if (HasDependentUsers) + promoteToReadySet(ReadyInstructions); } void Scheduler::promoteToReadySet(SmallVectorImpl &Ready) { @@ -376,21 +385,12 @@ InstRef Scheduler::select() { return InstRef(); // We found an instruction to issue. - InstRef IR = ReadySet[QueueIndex]; std::swap(ReadySet[QueueIndex], ReadySet[ReadySet.size() - 1]); ReadySet.pop_back(); return IR; } -void Scheduler::updatePendingQueue(SmallVectorImpl &Ready) { - // Notify to instructions in the pending queue that a new cycle just - // started. - for (InstRef &Entry : WaitSet) - Entry.getInstruction()->cycleEvent(); - promoteToReadySet(Ready); -} - void Scheduler::updateIssuedSet(SmallVectorImpl &Executed) { unsigned RemovedElements = 0; for (auto I = IssuedSet.begin(), E = IssuedSet.end(); I != E;) { @@ -398,7 +398,6 @@ void Scheduler::updateIssuedSet(SmallVectorImpl &Executed) { if (!IR.isValid()) break; Instruction &IS = *IR.getInstruction(); - IS.cycleEvent(); if (!IS.isExecuted()) { LLVM_DEBUG(dbgs() << "[SCHEDULER]: Instruction #" << IR << " is still executing.\n"); @@ -417,8 +416,22 @@ void Scheduler::updateIssuedSet(SmallVectorImpl &Executed) { IssuedSet.resize(IssuedSet.size() - RemovedElements); } -void Scheduler::reclaimSimulatedResources(SmallVectorImpl &Freed) { +void Scheduler::cycleEvent(SmallVectorImpl &Freed, + SmallVectorImpl &Executed, + SmallVectorImpl &Ready) { + // Release consumed resources. Resources->cycleEvent(Freed); + + // Propagate the cycle event to the 'Issued' and 'Wait' sets. + for (InstRef &IR : IssuedSet) + IR.getInstruction()->cycleEvent(); + + updateIssuedSet(Executed); + + for (InstRef &IR : WaitSet) + IR.getInstruction()->cycleEvent(); + + promoteToReadySet(Ready); } bool Scheduler::mustIssueImmediately(const InstRef &IR) const { diff --git a/tools/llvm-mca/Scheduler.h b/tools/llvm-mca/Scheduler.h index 1fc9b8d5192..341c66ea2ce 100644 --- a/tools/llvm-mca/Scheduler.h +++ b/tools/llvm-mca/Scheduler.h @@ -378,6 +378,15 @@ class Scheduler : public HardwareUnit { InstRef &IR, llvm::SmallVectorImpl> &Pipes); + // Identify instructions that have finished executing, and remove them from + // the IssuedSet. References to executed instructions are added to input + // vector 'Executed'. + void updateIssuedSet(llvm::SmallVectorImpl &Executed); + + // Try to promote instructions from WaitSet to ReadySet. + // Add promoted instructions to the 'Ready' vector in input. + void promoteToReadySet(llvm::SmallVectorImpl &Ready); + public: Scheduler(const llvm::MCSchedModel &Model, LSUnit *Lsu) : SM(Model), LSU(Lsu), Resources(llvm::make_unique(SM)) { @@ -414,39 +423,40 @@ public: /// This method assumes that IR has been previously dispatched. bool isReady(const InstRef &IR) const; - /// Issue an instruction. The Used container is populated with - /// the resource objects consumed on behalf of issuing this instruction. + /// Issue an instruction and populates a vector of used pipeline resources, + /// and a vector of instructions that transitioned to the ready state as a + /// result of this event. void issueInstruction(InstRef &IR, - llvm::SmallVectorImpl> &Used); + llvm::SmallVectorImpl> &Used, + llvm::SmallVectorImpl &Ready); /// Returns true if IR has to be issued immediately, or if IR is a zero /// latency instruction. bool mustIssueImmediately(const InstRef &IR) const; - /// Update the resources managed by the scheduler. - /// This routine is to be called at the start of a new cycle, and is - /// responsible for updating scheduler resources. Resources are released - /// once they have been fully consumed. - void reclaimSimulatedResources(llvm::SmallVectorImpl &Freed); + /// This routine notifies the Scheduler that a new cycle just started. + /// + /// It notifies the underlying ResourceManager that a new cycle just started. + /// Vector `Freed` is populated with resourceRef related to resources that + /// have changed in state, and that are now available to new instructions. + /// Instructions executed are added to vector Executed, while vector Ready is + /// populated with instructions that have become ready in this new cycle. + void cycleEvent(llvm::SmallVectorImpl &Freed, + llvm::SmallVectorImpl &Ready, + llvm::SmallVectorImpl &Executed); - /// Move instructions from the WaitSet to the ReadySet if input operands - /// are all available. - void promoteToReadySet(llvm::SmallVectorImpl &Ready); - - /// Update the ready queue. - void updatePendingQueue(llvm::SmallVectorImpl &Ready); - - /// Update the issued queue. - void updateIssuedSet(llvm::SmallVectorImpl &Executed); - - /// Obtain the processor's resource identifier for the given - /// resource mask. - unsigned getResourceID(uint64_t Mask) { + /// Convert a resource mask into a valid llvm processor resource identifier. + unsigned getResourceID(uint64_t Mask) const { return Resources->resolveResourceMask(Mask); } /// Select the next instruction to issue from the ReadySet. - /// This method gives priority to older instructions. + /// + /// The default implementation of this method ranks instructions based on + /// their age, and the number of known users. It prioritizes older + /// instructions over younger instructions to minimize the pressure on the + /// reorder buffer. It also gives a little priority boost to instructions + /// with multiple users to better expose ILP. InstRef select(); #ifndef NDEBUG