diff --git a/include/llvm/MCA/HardwareUnits/LSUnit.h b/include/llvm/MCA/HardwareUnits/LSUnit.h index c9e1cc3cdc4..e2ba9cbbf8b 100644 --- a/include/llvm/MCA/HardwareUnits/LSUnit.h +++ b/include/llvm/MCA/HardwareUnits/LSUnit.h @@ -18,13 +18,83 @@ #include "llvm/ADT/SmallSet.h" #include "llvm/MC/MCSchedule.h" #include "llvm/MCA/HardwareUnits/HardwareUnit.h" +#include "llvm/MCA/Instruction.h" namespace llvm { namespace mca { -class InstRef; class Scheduler; +/// Abstract base interface for LS (load/store) units in llvm-mca. +class LSUnitBase : public HardwareUnit { + /// Load queue size. + /// + /// A value of zero for this field means that the load queue is unbounded. + /// Processor models can declare the size of a load queue via tablegen (see + /// the definition of tablegen class LoadQueue in + /// llvm/Target/TargetSchedule.td). + unsigned LQSize; + + /// Load queue size. + /// + /// A value of zero for this field means that the store queue is unbounded. + /// Processor models can declare the size of a store queue via tablegen (see + /// the definition of tablegen class StoreQueue in + /// llvm/Target/TargetSchedule.td). + unsigned SQSize; + + /// True if loads don't alias with stores. + /// + /// By default, the LS unit assumes that loads and stores don't alias with + /// eachother. If this field is set to false, then loads are always assumed to + /// alias with stores. + const bool NoAlias; + +public: + LSUnitBase(const MCSchedModel &SM, unsigned LoadQueueSize, + unsigned StoreQueueSize, bool AssumeNoAlias); + + virtual ~LSUnitBase(); + + /// Returns the total number of entries in the load queue. + unsigned getLoadQueueSize() const { return LQSize; } + + /// Returns the total number of entries in the store queue. + unsigned getStoreQueueSize() const { return SQSize; } + + bool assumeNoAlias() const { return NoAlias; } + + enum Status { + LSU_AVAILABLE = 0, + LSU_LQUEUE_FULL, // Load Queue unavailable + LSU_SQUEUE_FULL // Store Queue unavailable + }; + + /// This method checks the availability of the load/store buffers. + /// + /// Returns LSU_AVAILABLE if there are enough load/store queue entries to + /// accomodate instruction IR. By default, LSU_AVAILABLE is returned if IR is + /// not a memory operation. + virtual Status isAvailable(const InstRef &IR) const = 0; + + /// Allocates LS resources for instruction IR. + /// + /// This method assumes that a previous call to `isAvailable(IR)` succeeded + /// with a LSUnitBase::Status value of LSU_AVAILABLE. + virtual void dispatch(const InstRef &IR) = 0; + + /// Check if a peviously dispatched instruction IR is now ready for execution. + /// + /// Instruction IR is assumed to be a memory operation. If IR is still waiting + /// on another memory instruction M, then M is returned to the caller. If IR + /// depends on more than one memory operations, then this method returns one + /// of them. + /// + /// Derived classes can implement memory consistency rules for simulated + /// processor within this member function. + virtual const InstRef &isReady(const InstRef &IR) const = 0; +}; + /// A Load/Store Unit implementing a load and store queues. /// /// This class implements a load queue and a store queue to emulate the @@ -88,18 +158,7 @@ class Scheduler; /// A load/store barrier is "executed" when it becomes the oldest entry in /// the load/store queue(s). That also means, all the older loads/stores have /// already been executed. -class LSUnit : public HardwareUnit { - // Load queue size. - // LQ_Size == 0 means that there are infinite slots in the load queue. - unsigned LQ_Size; - - // Store queue size. - // SQ_Size == 0 means that there are infinite slots in the store queue. - unsigned SQ_Size; - - // If true, loads will never alias with stores. This is the default. - bool NoAlias; - +class LSUnit : public LSUnitBase { // When a `MayLoad` instruction is dispatched to the schedulers for execution, // the LSUnit reserves an entry in the `LoadQueue` for it. // @@ -138,68 +197,75 @@ class LSUnit : public HardwareUnit { // alternative approaches that let instructions specify the number of // load/store queue entries which they consume at dispatch stage (See // PR39830). - SmallSet LoadQueue; - SmallSet StoreQueue; + SmallSet LoadQueue; + SmallSet StoreQueue; - void assignLQSlot(unsigned Index); - void assignSQSlot(unsigned Index); + void assignLQSlot(const InstRef &IR); + void assignSQSlot(const InstRef &IR); // An instruction that both 'mayStore' and 'HasUnmodeledSideEffects' is // conservatively treated as a store barrier. It forces older store to be // executed before newer stores are issued. - SmallSet StoreBarriers; + SmallSet StoreBarriers; // An instruction that both 'MayLoad' and 'HasUnmodeledSideEffects' is // conservatively treated as a load barrier. It forces older loads to execute // before newer loads are issued. - SmallSet LoadBarriers; + SmallSet LoadBarriers; bool isSQEmpty() const { return StoreQueue.empty(); } bool isLQEmpty() const { return LoadQueue.empty(); } - bool isSQFull() const { return SQ_Size != 0 && StoreQueue.size() == SQ_Size; } - bool isLQFull() const { return LQ_Size != 0 && LoadQueue.size() == LQ_Size; } + bool isSQFull() const { + return getStoreQueueSize() != 0 && StoreQueue.size() == getStoreQueueSize(); + } + bool isLQFull() const { + return getLoadQueueSize() != 0 && LoadQueue.size() == getLoadQueueSize(); + } public: - LSUnit(const MCSchedModel &SM, unsigned LQ = 0, unsigned SQ = 0, - bool AssumeNoAlias = false); + LSUnit(const MCSchedModel &SM) + : LSUnit(SM, /* LQSize */ 0, /* SQSize */ 0, /* NoAlias */ false) {} + LSUnit(const MCSchedModel &SM, unsigned LQ, unsigned SQ) + : LSUnit(SM, LQ, SQ, /* NoAlias */ false) {} + LSUnit(const MCSchedModel &SM, unsigned LQ, unsigned SQ, bool AssumeNoAlias) + : LSUnitBase(SM, LQ, SQ, AssumeNoAlias) {} #ifndef NDEBUG void dump() const; #endif - enum Status { LSU_AVAILABLE = 0, LSU_LQUEUE_FULL, LSU_SQUEUE_FULL }; + /// Returns LSU_AVAILABLE if there are enough load/store queue entries to + /// accomodate instruction IR. + Status isAvailable(const InstRef &IR) const override; - // Returns LSU_AVAILABLE if there are enough load/store queue entries to serve - // IR. It also returns LSU_AVAILABLE if IR is not a memory operation. - Status isAvailable(const InstRef &IR) const; + /// Allocates LS resources for instruction IR. + /// + /// This method assumes that a previous call to `isAvailable(IR)` succeeded + /// returning LSU_AVAILABLE. + void dispatch(const InstRef &IR) override; - // Allocates load/store queue resources for IR. - // - // This method assumes that a previous call to `isAvailable(IR)` returned - // LSU_AVAILABLE, and that IR is a memory operation. - void dispatch(const InstRef &IR); + /// Check if a peviously dispatched instruction IR is now ready for execution. + /// + /// Rules are: + /// By default, rules are: + /// 1. A store may not pass a previous store. + /// 2. A load may not pass a previous store unless flag 'NoAlias' is set. + /// 3. A load may pass a previous load. + /// 4. A store may not pass a previous load (regardless of flag 'NoAlias'). + /// 5. A load has to wait until an older load barrier is fully executed. + /// 6. A store has to wait until an older store barrier is fully executed. + const InstRef &isReady(const InstRef &IR) const override; - // By default, rules are: - // 1. A store may not pass a previous store. - // 2. A load may not pass a previous store unless flag 'NoAlias' is set. - // 3. A load may pass a previous load. - // 4. A store may not pass a previous load (regardless of flag 'NoAlias'). - // 5. A load has to wait until an older load barrier is fully executed. - // 6. A store has to wait until an older store barrier is fully executed. - // - // Returns an instruction identifier. If IR is ready, then this method returns - // `IR.getSourceIndex()`. Otherwise it returns the instruction ID of the - // dependent (i.e. conflicting) memory instruction. - virtual unsigned isReady(const InstRef &IR) const; - - // Load and store instructions are tracked by their corresponding queues from - // dispatch until the "instruction executed" event. - // Only when a load instruction reaches the 'Executed' stage, its value - // becomes available to the users. At that point, the load no longer needs to - // be tracked by the load queue. - // FIXME: For simplicity, we optimistically assume a similar behavior for - // store instructions. In practice, store operations don't tend to leave the - // store queue until they reach the 'Retired' stage (See PR39830). + /// Instruction executed event handler. + /// + /// Load and store instructions are tracked by their corresponding queues from + /// dispatch until "instruction executed" event. + /// When a load instruction Ld reaches the 'Executed' stage, its value + /// is propagated to all the dependent users, and the LS unit stops tracking + /// Ld. + /// FIXME: For simplicity, we optimistically assume a similar behavior for + /// store instructions. In practice, store operations don't tend to leave the + /// store queue until they reach the 'Retired' stage (See PR39830). void onInstructionExecuted(const InstRef &IR); }; diff --git a/lib/MCA/HardwareUnits/LSUnit.cpp b/lib/MCA/HardwareUnits/LSUnit.cpp index 4f49fbd2bb4..c3866d6bba7 100644 --- a/lib/MCA/HardwareUnits/LSUnit.cpp +++ b/lib/MCA/HardwareUnits/LSUnit.cpp @@ -21,48 +21,48 @@ namespace llvm { namespace mca { -LSUnit::LSUnit(const MCSchedModel &SM, unsigned LQ, unsigned SQ, - bool AssumeNoAlias) - : LQ_Size(LQ), SQ_Size(SQ), NoAlias(AssumeNoAlias) { +LSUnitBase::LSUnitBase(const MCSchedModel &SM, unsigned LQ, unsigned SQ, + bool AssumeNoAlias) + : LQSize(LQ), SQSize(SQ), NoAlias(AssumeNoAlias) { if (SM.hasExtraProcessorInfo()) { const MCExtraProcessorInfo &EPI = SM.getExtraProcessorInfo(); - if (!LQ_Size && EPI.LoadQueueID) { + if (!LQSize && EPI.LoadQueueID) { const MCProcResourceDesc &LdQDesc = *SM.getProcResource(EPI.LoadQueueID); - LQ_Size = LdQDesc.BufferSize; + LQSize = LdQDesc.BufferSize; } - if (!SQ_Size && EPI.StoreQueueID) { + if (!SQSize && EPI.StoreQueueID) { const MCProcResourceDesc &StQDesc = *SM.getProcResource(EPI.StoreQueueID); - SQ_Size = StQDesc.BufferSize; + SQSize = StQDesc.BufferSize; } } } +LSUnitBase::~LSUnitBase() {} + #ifndef NDEBUG void LSUnit::dump() const { - dbgs() << "[LSUnit] LQ_Size = " << LQ_Size << '\n'; - dbgs() << "[LSUnit] SQ_Size = " << SQ_Size << '\n'; + dbgs() << "[LSUnit] LQ_Size = " << getLoadQueueSize() << '\n'; + dbgs() << "[LSUnit] SQ_Size = " << getStoreQueueSize() << '\n'; dbgs() << "[LSUnit] NextLQSlotIdx = " << LoadQueue.size() << '\n'; dbgs() << "[LSUnit] NextSQSlotIdx = " << StoreQueue.size() << '\n'; } #endif -void LSUnit::assignLQSlot(unsigned Index) { - assert(!isLQFull()); - assert(LoadQueue.count(Index) == 0); +void LSUnit::assignLQSlot(const InstRef &IR) { + assert(!isLQFull() && "Load Queue is full!"); - LLVM_DEBUG(dbgs() << "[LSUnit] - AssignLQSlot \n"); - LoadQueue.insert(Index); + LoadQueue.insert(IR); } -void LSUnit::assignSQSlot(unsigned Index) { - assert(!isSQFull()); - assert(StoreQueue.count(Index) == 0); +void LSUnit::assignSQSlot(const InstRef &IR) { + assert(!isSQFull() && "Store Queue is full!"); - LLVM_DEBUG(dbgs() << "[LSUnit] - AssignSQSlot \n"); - StoreQueue.insert(Index); + StoreQueue.insert(IR); } void LSUnit::dispatch(const InstRef &IR) { @@ -70,17 +70,16 @@ void LSUnit::dispatch(const InstRef &IR) { unsigned IsMemBarrier = Desc.HasSideEffects; assert((Desc.MayLoad || Desc.MayStore) && "Not a memory operation!"); - const unsigned Index = IR.getSourceIndex(); if (Desc.MayLoad) { if (IsMemBarrier) - LoadBarriers.insert(Index); - assignLQSlot(Index); + LoadBarriers.insert(IR); + assignLQSlot(IR); } if (Desc.MayStore) { if (IsMemBarrier) - StoreBarriers.insert(Index); - assignSQSlot(Index); + StoreBarriers.insert(IR); + assignSQSlot(IR); } } @@ -93,65 +92,67 @@ LSUnit::Status LSUnit::isAvailable(const InstRef &IR) const { return LSUnit::LSU_AVAILABLE; } -unsigned LSUnit::isReady(const InstRef &IR) const { +const InstRef &LSUnit::isReady(const InstRef &IR) const { const InstrDesc &Desc = IR.getInstruction()->getDesc(); const unsigned Index = IR.getSourceIndex(); bool IsALoad = Desc.MayLoad; bool IsAStore = Desc.MayStore; assert((IsALoad || IsAStore) && "Not a memory operation!"); - assert((!IsALoad || LoadQueue.count(Index) == 1) && "Load not in queue!"); - assert((!IsAStore || StoreQueue.count(Index) == 1) && "Store not in queue!"); if (IsALoad && !LoadBarriers.empty()) { - unsigned LoadBarrierIndex = *LoadBarriers.begin(); + const InstRef &LoadBarrier = *LoadBarriers.begin(); // A younger load cannot pass a older load barrier. - if (Index > LoadBarrierIndex) - return LoadBarrierIndex; + if (Index > LoadBarrier.getSourceIndex()) + return LoadBarrier; // A load barrier cannot pass a older load. - if (Index == LoadBarrierIndex && Index != *LoadQueue.begin()) - return *LoadQueue.begin(); + if (Index == LoadBarrier.getSourceIndex()) { + const InstRef &Load = *LoadQueue.begin(); + if (Index != Load.getSourceIndex()) + return Load; + } } if (IsAStore && !StoreBarriers.empty()) { - unsigned StoreBarrierIndex = *StoreBarriers.begin(); + const InstRef &StoreBarrier = *StoreBarriers.begin(); // A younger store cannot pass a older store barrier. - if (Index > StoreBarrierIndex) - return StoreBarrierIndex; + if (Index > StoreBarrier.getSourceIndex()) + return StoreBarrier; // A store barrier cannot pass a older store. - if (Index == StoreBarrierIndex && Index != *StoreQueue.begin()) - return *StoreQueue.begin(); + if (Index == StoreBarrier.getSourceIndex()) { + const InstRef &Store = *StoreQueue.begin(); + if (Index != Store.getSourceIndex()) + return Store; + } } // A load may not pass a previous store unless flag 'NoAlias' is set. // A load may pass a previous load. - if (NoAlias && IsALoad) - return Index; + if (assumeNoAlias() && IsALoad) + return IR; if (StoreQueue.size()) { // A load may not pass a previous store. // A store may not pass a previous store. - if (Index > *StoreQueue.begin()) - return *StoreQueue.begin(); + const InstRef &Store = *StoreQueue.begin(); + if (Index > Store.getSourceIndex()) + return Store; } // Okay, we are older than the oldest store in the queue. - // If there are no pending loads, then we can say for sure that this - // instruction is ready. if (isLQEmpty()) - return Index; + return IR; // Check if there are no older loads. - if (Index <= *LoadQueue.begin()) - return Index; + const InstRef &Load = *LoadQueue.begin(); + if (Index <= Load.getSourceIndex()) + return IR; - // There is at least one younger load. - // // A load may pass a previous load. if (IsALoad) - return Index; + return IR; // A store may not pass a previous load. - return *LoadQueue.begin(); + return Load; } void LSUnit::onInstructionExecuted(const InstRef &IR) { @@ -161,29 +162,35 @@ void LSUnit::onInstructionExecuted(const InstRef &IR) { bool IsAStore = Desc.MayStore; if (IsALoad) { - if (LoadQueue.erase(Index)) { + if (LoadQueue.erase(IR)) { LLVM_DEBUG(dbgs() << "[LSUnit]: Instruction idx=" << Index << " has been removed from the load queue.\n"); } - if (!LoadBarriers.empty() && Index == *LoadBarriers.begin()) { - LLVM_DEBUG( - dbgs() << "[LSUnit]: Instruction idx=" << Index - << " has been removed from the set of load barriers.\n"); - LoadBarriers.erase(Index); + if (!LoadBarriers.empty()) { + const InstRef &LoadBarrier = *LoadBarriers.begin(); + if (Index == LoadBarrier.getSourceIndex()) { + LLVM_DEBUG( + dbgs() << "[LSUnit]: Instruction idx=" << Index + << " has been removed from the set of load barriers.\n"); + LoadBarriers.erase(IR); + } } } if (IsAStore) { - if (StoreQueue.erase(Index)) { + if (StoreQueue.erase(IR)) { LLVM_DEBUG(dbgs() << "[LSUnit]: Instruction idx=" << Index << " has been removed from the store queue.\n"); } - if (!StoreBarriers.empty() && Index == *StoreBarriers.begin()) { - LLVM_DEBUG( - dbgs() << "[LSUnit]: Instruction idx=" << Index - << " has been removed from the set of store barriers.\n"); - StoreBarriers.erase(Index); + if (!StoreBarriers.empty()) { + const InstRef &StoreBarrier = *StoreBarriers.begin(); + if (Index == StoreBarrier.getSourceIndex()) { + LLVM_DEBUG( + dbgs() << "[LSUnit]: Instruction idx=" << Index + << " has been removed from the set of store barriers.\n"); + StoreBarriers.erase(IR); + } } } } diff --git a/lib/MCA/HardwareUnits/Scheduler.cpp b/lib/MCA/HardwareUnits/Scheduler.cpp index 9eeea9d0113..bf48d928899 100644 --- a/lib/MCA/HardwareUnits/Scheduler.cpp +++ b/lib/MCA/HardwareUnits/Scheduler.cpp @@ -119,9 +119,9 @@ bool Scheduler::promoteToReadySet(SmallVectorImpl &Ready) { // Check if there are still unsolved memory dependencies. Instruction &IS = *IR.getInstruction(); if (IS.isMemOp()) { - unsigned CriticalMemDep = LSU.isReady(IR); - if (CriticalMemDep != IR.getSourceIndex()) { - IS.setCriticalMemDep(CriticalMemDep); + const InstRef &CriticalMemDep = LSU.isReady(IR); + if (CriticalMemDep != IR) { + IS.setCriticalMemDep(CriticalMemDep.getSourceIndex()); ++I; continue; } @@ -158,7 +158,7 @@ bool Scheduler::promoteToPendingSet(SmallVectorImpl &Pending) { break; // Check if this instruction is now ready. In case, force - // a transition in state using method 'update()'. + // a transition in state using method 'updateDispatched()'. Instruction &IS = *IR.getInstruction(); if (IS.isDispatched() && !IS.updateDispatched()) { ++I; @@ -242,12 +242,10 @@ void Scheduler::analyzeDataDependencies(SmallVectorImpl &RegDeps, if (Resources->checkAvailability(IS.getDesc())) continue; - if (IS.isReady() || - (IS.isMemOp() && LSU.isReady(IR) != IR.getSourceIndex())) { + if (IS.isReady() || (IS.isMemOp() && LSU.isReady(IR) != IR)) MemDeps.emplace_back(IR); - } else { + else RegDeps.emplace_back(IR); - } } } @@ -304,8 +302,7 @@ bool Scheduler::dispatch(const InstRef &IR) { // Memory operations that are not in a ready state are initially assigned to // the WaitSet. - if (!IS.isReady() || - (IS.isMemOp() && LSU.isReady(IR) != IR.getSourceIndex())) { + if (!IS.isReady() || (IS.isMemOp() && LSU.isReady(IR) != IR)) { LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR << " to the WaitSet\n"); WaitSet.push_back(IR); return false;