diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h index 2645f8113e7..a7edaaa9067 100644 --- a/include/llvm/CodeGen/MachineScheduler.h +++ b/include/llvm/CodeGen/MachineScheduler.h @@ -234,7 +234,7 @@ public: virtual void leaveMBB() {}; /// Notify this strategy that all roots have been released (including those - /// that depend on ExitSU). + /// that depend on EntrySU or ExitSU). virtual void registerRoots() {} /// Pick the next node to schedule, or return NULL. Set IsTopNode to true to diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h index 6084a620d5a..4c8d047727c 100644 --- a/include/llvm/CodeGen/ScheduleDAG.h +++ b/include/llvm/CodeGen/ScheduleDAG.h @@ -560,6 +560,7 @@ class TargetRegisterInfo; MachineFunction &MF; ///< Machine function MachineRegisterInfo &MRI; ///< Virtual/real register map std::vector SUnits; ///< The scheduling units. + SUnit EntrySU; ///< Special node for the region entry. SUnit ExitSU; ///< Special node for the region exit. #ifdef NDEBUG diff --git a/lib/CodeGen/MachineScheduler.cpp b/lib/CodeGen/MachineScheduler.cpp index dbb2f3f34ce..b6d0d9a74ac 100644 --- a/lib/CodeGen/MachineScheduler.cpp +++ b/lib/CodeGen/MachineScheduler.cpp @@ -680,7 +680,7 @@ void ScheduleDAGMI::releasePred(SUnit *SU, SDep *PredEdge) { PredSU->BotReadyCycle = SU->BotReadyCycle + PredEdge->getLatency(); --PredSU->NumSuccsLeft; - if (PredSU->NumSuccsLeft == 0) + if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) SchedImpl->releaseBottomNode(PredSU); } @@ -853,7 +853,7 @@ void ScheduleDAGMI::initQueues(ArrayRef TopRoots, NextClusterSucc = nullptr; NextClusterPred = nullptr; - // Release all DAG roots for scheduling, not including ExitSU. + // Release all DAG roots for scheduling, not including EntrySU/ExitSU. // // Nodes with unreleased weak edges can still be roots. // Release top roots in forward order. @@ -867,6 +867,7 @@ void ScheduleDAGMI::initQueues(ArrayRef TopRoots, SchedImpl->releaseBottomNode(*I); } + releaseSuccessors(&EntrySU); releasePredecessors(&ExitSU); SchedImpl->registerRoots(); @@ -1167,6 +1168,8 @@ void ScheduleDAGMILive::updatePressureDiffs( void ScheduleDAGMILive::dump() const { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + if (EntrySU.getInstr() != nullptr) + dumpNodeAll(EntrySU); for (const SUnit &SU : SUnits) { dumpNodeAll(SU); if (ShouldTrackPressure) { diff --git a/lib/CodeGen/MacroFusion.cpp b/lib/CodeGen/MacroFusion.cpp index 4243dcfd89c..d2ee21c8720 100644 --- a/lib/CodeGen/MacroFusion.cpp +++ b/lib/CodeGen/MacroFusion.cpp @@ -109,21 +109,23 @@ static bool fuseInstructionPair(ScheduleDAGInstrs &DAG, SUnit &FirstSU, // Make the FirstSU also dependent on the dependencies of the SecondSU to // prevent them from being scheduled between the FirstSU and the SecondSU. - for (const SDep &SI : SecondSU.Preds) { - SUnit *SU = SI.getSUnit(); - if (SI.isWeak() || isHazard(SI) || &FirstSU == SU || FirstSU.isSucc(SU)) - continue; - LLVM_DEBUG(dbgs() << " Bind "; DAG.dumpNodeName(*SU); dbgs() << " - "; - DAG.dumpNodeName(FirstSU); dbgs() << '\n';); - DAG.addEdge(&FirstSU, SDep(SU, SDep::Artificial)); - } - // ExitSU comes last by design, which acts like an implicit dependency - // between ExitSU and any bottom root in the graph. We should transfer - // this to FirstSU as well. - if (&SecondSU == &DAG.ExitSU) { - for (SUnit &SU : DAG.SUnits) { - if (SU.Succs.empty()) - DAG.addEdge(&FirstSU, SDep(&SU, SDep::Artificial)); + if (&FirstSU != &DAG.EntrySU) { + for (const SDep &SI : SecondSU.Preds) { + SUnit *SU = SI.getSUnit(); + if (SI.isWeak() || isHazard(SI) || &FirstSU == SU || FirstSU.isSucc(SU)) + continue; + LLVM_DEBUG(dbgs() << " Bind "; DAG.dumpNodeName(*SU); dbgs() << " - "; + DAG.dumpNodeName(FirstSU); dbgs() << '\n';); + DAG.addEdge(&FirstSU, SDep(SU, SDep::Artificial)); + } + // ExitSU comes last by design, which acts like an implicit dependency + // between ExitSU and any bottom root in the graph. We should transfer + // this to FirstSU as well. + if (&SecondSU == &DAG.ExitSU) { + for (SUnit &SU : DAG.SUnits) { + if (SU.Succs.empty()) + DAG.addEdge(&FirstSU, SDep(&SU, SDep::Artificial)); + } } } diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp index d6af43792e5..b85f00a61ea 100644 --- a/lib/CodeGen/PostRASchedulerList.cpp +++ b/lib/CodeGen/PostRASchedulerList.cpp @@ -527,6 +527,9 @@ void SchedulePostRATDList::ListScheduleTopDown() { // blocks are a single region). HazardRec->Reset(); + // Release any successors of the special Entry node. + ReleaseSuccessors(&EntrySU); + // Add all leaves to Available queue. for (unsigned i = 0, e = SUnits.size(); i != e; ++i) { // It is available if it has no predecessors. diff --git a/lib/CodeGen/ScheduleDAG.cpp b/lib/CodeGen/ScheduleDAG.cpp index 0e9c4e6747a..60f8eec1b9b 100644 --- a/lib/CodeGen/ScheduleDAG.cpp +++ b/lib/CodeGen/ScheduleDAG.cpp @@ -63,6 +63,7 @@ ScheduleDAG::~ScheduleDAG() = default; void ScheduleDAG::clearDAG() { SUnits.clear(); + EntrySU = SUnit(); ExitSU = SUnit(); } @@ -351,7 +352,9 @@ LLVM_DUMP_METHOD void SUnit::dumpAttributes() const { } LLVM_DUMP_METHOD void ScheduleDAG::dumpNodeName(const SUnit &SU) const { - if (&SU == &ExitSU) + if (&SU == &EntrySU) + dbgs() << "EntrySU"; + else if (&SU == &ExitSU) dbgs() << "ExitSU"; else dbgs() << "SU(" << SU.NodeNum << ")"; @@ -653,7 +656,7 @@ std::vector ScheduleDAGTopologicalSort::GetSubGraph(const SUnit &StartSU, for (int I = SU->Preds.size()-1; I >= 0; --I) { const SUnit *Pred = SU->Preds[I].getSUnit(); unsigned s = Pred->NodeNum; - // Edges to non-SUnits are allowed but ignored (e.g. ExitSU). + // Edges to non-SUnits are allowed but ignored (e.g. EntrySU). if (Pred->isBoundaryNode()) continue; if (Node2Index[s] == LowerBound) { diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp index 9fb24eda6f6..10453c4c767 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -1167,6 +1167,8 @@ void ScheduleDAGInstrs::dumpNode(const SUnit &SU) const { void ScheduleDAGInstrs::dump() const { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + if (EntrySU.getInstr() != nullptr) + dumpNodeAll(EntrySU); for (const SUnit &SU : SUnits) dumpNodeAll(SU); if (ExitSU.getInstr() != nullptr) @@ -1177,7 +1179,9 @@ void ScheduleDAGInstrs::dump() const { std::string ScheduleDAGInstrs::getGraphNodeLabel(const SUnit *SU) const { std::string s; raw_string_ostream oss(s); - if (SU == &ExitSU) + if (SU == &EntrySU) + oss << ""; + else if (SU == &ExitSU) oss << ""; else SU->getInstr()->print(oss, /*IsStandalone=*/true); diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp index e2a9d7c4872..2902c96c765 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp @@ -150,8 +150,8 @@ void ScheduleDAGFast::ReleasePred(SUnit *SU, SDep *PredEdge) { --PredSU->NumSuccsLeft; // If all the node's successors are scheduled, this node is ready - // to be scheduled. - if (PredSU->NumSuccsLeft == 0) { + // to be scheduled. Ignore the special EntrySU node. + if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) { PredSU->isAvailable = true; AvailableQueue.push(PredSU); } diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index 73f09e6111e..7a5e8ac6075 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -415,8 +415,8 @@ void ScheduleDAGRRList::ReleasePred(SUnit *SU, const SDep *PredEdge) { } // If all the node's successors are scheduled, this node is ready - // to be scheduled. - if (PredSU->NumSuccsLeft == 0) { + // to be scheduled. Ignore the special EntrySU node. + if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) { PredSU->isAvailable = true; unsigned Height = PredSU->getHeight(); diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index db9f4e37594..76fb0340a7a 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -696,6 +696,8 @@ void ScheduleDAGSDNodes::dumpNode(const SUnit &SU) const { void ScheduleDAGSDNodes::dump() const { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + if (EntrySU.getNode() != nullptr) + dumpNodeAll(EntrySU); for (const SUnit &SU : SUnits) dumpNodeAll(SU); if (ExitSU.getNode() != nullptr) diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp index 05d3eaa6c64..e7bac73678a 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp @@ -166,6 +166,9 @@ void ScheduleDAGVLIW::scheduleNodeTopDown(SUnit *SU, unsigned CurCycle) { void ScheduleDAGVLIW::listScheduleTopDown() { unsigned CurCycle = 0; + // Release any successors of the special Entry node. + releaseSuccessors(&EntrySU); + // All leaves to AvailableQueue. for (unsigned i = 0, e = SUnits.size(); i != e; ++i) { // It is available if it has no predecessors. diff --git a/lib/Target/AMDGPU/GCNMinRegStrategy.cpp b/lib/Target/AMDGPU/GCNMinRegStrategy.cpp index 417abc4b5ce..884b2e17289 100644 --- a/lib/Target/AMDGPU/GCNMinRegStrategy.cpp +++ b/lib/Target/AMDGPU/GCNMinRegStrategy.cpp @@ -248,6 +248,7 @@ GCNMinRegScheduler::schedule(ArrayRef TopRoots, for (auto SU : TopRoots) { RQ.push_back(*new (Alloc.Allocate()) Candidate(SU, StepNo)); } + releaseSuccessors(&DAG.EntrySU, StepNo); while (!RQ.empty()) { LLVM_DEBUG(dbgs() << "\n=== Picking candidate, Step = " << StepNo diff --git a/lib/Target/AMDGPU/SIMachineScheduler.h b/lib/Target/AMDGPU/SIMachineScheduler.h index cfa6734cdb5..0a8abbb47de 100644 --- a/lib/Target/AMDGPU/SIMachineScheduler.h +++ b/lib/Target/AMDGPU/SIMachineScheduler.h @@ -455,6 +455,7 @@ public: MachineRegisterInfo *getMRI() { return &MRI; } const TargetRegisterInfo *getTRI() { return TRI; } ScheduleDAGTopologicalSort *GetTopo() { return &Topo; } + SUnit &getEntrySU() { return EntrySU; } SUnit& getExitSU() { return ExitSU; } void restoreSULinksLeft();