mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2024-11-23 03:02:36 +01:00
[Hexagon] Refactor HexagonShuffle
The check() in HexagonShuffle has been decomposed into smaller steps. No functionality change is intended with this commit.
This commit is contained in:
Brian Cain
Krzysztof Parzyszek
parent
e480c416f6
commit
f0c4115585
@ -207,6 +207,7 @@ HexagonShuffler::HexagonShuffler(MCContext &Context, bool ReportErrors,
|
||||
void HexagonShuffler::reset() {
|
||||
Packet.clear();
|
||||
BundleFlags = 0;
|
||||
CheckFailure = false;
|
||||
}
|
||||
|
||||
void HexagonShuffler::append(MCInst const &ID, MCInst const *Extender,
|
||||
@ -216,386 +217,144 @@ void HexagonShuffler::append(MCInst const &ID, MCInst const *Extender,
|
||||
Packet.push_back(PI);
|
||||
}
|
||||
|
||||
static struct {
|
||||
const static struct {
|
||||
unsigned first;
|
||||
unsigned second;
|
||||
} jumpSlots[] = {{8, 4}, {8, 2}, {8, 1}, {4, 2}, {4, 1}, {2, 1}};
|
||||
#define MAX_JUMP_SLOTS (sizeof(jumpSlots) / sizeof(jumpSlots[0]))
|
||||
|
||||
void HexagonShuffler::restrictSlot1AOK() {
|
||||
bool HasRestrictSlot1AOK = false;
|
||||
SMLoc RestrictLoc;
|
||||
for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
|
||||
MCInst const &Inst = ISJ->getDesc();
|
||||
if (HexagonMCInstrInfo::isRestrictSlot1AOK(MCII, Inst)) {
|
||||
HasRestrictSlot1AOK = true;
|
||||
RestrictLoc = Inst.getLoc();
|
||||
}
|
||||
}
|
||||
if (HasRestrictSlot1AOK)
|
||||
for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
|
||||
MCInst const &Inst = ISJ->getDesc();
|
||||
unsigned Type = HexagonMCInstrInfo::getType(MCII, Inst);
|
||||
static const unsigned Slot0Mask = 1 << 0;
|
||||
static const unsigned Slot1Mask = 1 << 1;
|
||||
static const unsigned Slot3Mask = 1 << 3;
|
||||
static const unsigned slotSingleLoad = Slot0Mask;
|
||||
static const unsigned slotSingleStore = Slot0Mask;
|
||||
|
||||
void HexagonShuffler::restrictSlot1AOK(HexagonPacketSummary const &Summary) {
|
||||
if (Summary.Slot1AOKLoc)
|
||||
for (HexagonInstr &ISJ : insts()) {
|
||||
MCInst const &Inst = ISJ.getDesc();
|
||||
const unsigned Type = HexagonMCInstrInfo::getType(MCII, Inst);
|
||||
if (Type != HexagonII::TypeALU32_2op &&
|
||||
Type != HexagonII::TypeALU32_3op &&
|
||||
Type != HexagonII::TypeALU32_ADDI) {
|
||||
unsigned Units = ISJ->Core.getUnits();
|
||||
if (Units & 2U) {
|
||||
const unsigned Units = ISJ.Core.getUnits();
|
||||
|
||||
if (Units & Slot1Mask) {
|
||||
AppliedRestrictions.push_back(std::make_pair(
|
||||
Inst.getLoc(),
|
||||
"Instruction was restricted from being in slot 1"));
|
||||
AppliedRestrictions.push_back(
|
||||
std::make_pair(RestrictLoc, "Instruction can only be combine "
|
||||
"with an ALU instruction in slot 1"));
|
||||
ISJ->Core.setUnits(Units & ~2U);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void HexagonShuffler::restrictNoSlot1Store() {
|
||||
bool HasRestrictNoSlot1Store = false;
|
||||
SMLoc RestrictLoc;
|
||||
for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
|
||||
MCInst const &Inst = ISJ->getDesc();
|
||||
if (HexagonMCInstrInfo::isRestrictNoSlot1Store(MCII, Inst)) {
|
||||
HasRestrictNoSlot1Store = true;
|
||||
RestrictLoc = Inst.getLoc();
|
||||
}
|
||||
}
|
||||
if (HasRestrictNoSlot1Store) {
|
||||
bool AppliedRestriction = false;
|
||||
for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
|
||||
MCInst const &Inst = ISJ->getDesc();
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, Inst).mayStore()) {
|
||||
unsigned Units = ISJ->Core.getUnits();
|
||||
if (Units & 2U) {
|
||||
AppliedRestriction = true;
|
||||
AppliedRestrictions.push_back(std::make_pair(
|
||||
Inst.getLoc(),
|
||||
"Instruction was restricted from being in slot 1"));
|
||||
ISJ->Core.setUnits(Units & ~2U);
|
||||
*Summary.Slot1AOKLoc, "Instruction can only be combined "
|
||||
"with an ALU instruction in slot 1"));
|
||||
ISJ.Core.setUnits(Units & ~Slot1Mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (AppliedRestriction)
|
||||
AppliedRestrictions.push_back(std::make_pair(
|
||||
RestrictLoc, "Instruction does not allow a store in slot 1"));
|
||||
}
|
||||
}
|
||||
|
||||
void HexagonShuffler::applySlotRestrictions() {
|
||||
restrictSlot1AOK();
|
||||
restrictNoSlot1Store();
|
||||
void HexagonShuffler::restrictNoSlot1Store(
|
||||
HexagonPacketSummary const &Summary) {
|
||||
// If this packet contains an instruction that bars slot-1 stores,
|
||||
// we should mask off slot 1 from all of the store instructions in
|
||||
// this packet.
|
||||
|
||||
if (!Summary.NoSlot1StoreLoc)
|
||||
return;
|
||||
|
||||
bool AppliedRestriction = false;
|
||||
|
||||
for (HexagonInstr &ISJ : insts()) {
|
||||
MCInst const &Inst = ISJ.getDesc();
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, Inst).mayStore()) {
|
||||
unsigned Units = ISJ.Core.getUnits();
|
||||
if (Units & Slot1Mask) {
|
||||
AppliedRestriction = true;
|
||||
AppliedRestrictions.push_back(std::make_pair(
|
||||
Inst.getLoc(), "Instruction was restricted from being in slot 1"));
|
||||
ISJ.Core.setUnits(Units & ~Slot1Mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (AppliedRestriction)
|
||||
AppliedRestrictions.push_back(
|
||||
std::make_pair(*Summary.NoSlot1StoreLoc,
|
||||
"Instruction does not allow a store in slot 1"));
|
||||
}
|
||||
|
||||
/// Check that the packet is legal and enforce relative insn order.
|
||||
bool HexagonShuffler::check() {
|
||||
// Descriptive slot masks.
|
||||
const unsigned slotSingleLoad = 0x1, slotSingleStore = 0x1,
|
||||
slotThree = 0x8, // slotFirstJump = 0x8,
|
||||
slotFirstLoadStore = 0x2, slotLastLoadStore = 0x1;
|
||||
// Highest slots for branches and stores used to keep their original order.
|
||||
// unsigned slotJump = slotFirstJump;
|
||||
unsigned slotLoadStore = slotFirstLoadStore;
|
||||
// Number of memory operations, loads, solo loads, stores, solo stores, single
|
||||
// stores.
|
||||
unsigned memory = 0, loads = 0, load0 = 0, stores = 0, store0 = 0, store1 = 0;
|
||||
unsigned NonZCVIloads = 0, AllCVIloads = 0, CVIstores = 0;
|
||||
// Number of duplex insns
|
||||
unsigned duplex = 0;
|
||||
unsigned pSlot3Cnt = 0;
|
||||
unsigned memops = 0;
|
||||
iterator slot3ISJ = end();
|
||||
std::vector<iterator> foundBranches;
|
||||
unsigned reservedSlots = 0;
|
||||
bool HexagonShuffler::applySlotRestrictions(
|
||||
HexagonPacketSummary const &Summary) {
|
||||
// These restrictions can modify the slot masks in the instructions
|
||||
// in the Packet member. They should run unconditionally and their
|
||||
// order does not matter.
|
||||
restrictSlot1AOK(Summary);
|
||||
restrictNoSlot1Store(Summary);
|
||||
|
||||
// Collect information from the insns in the packet.
|
||||
for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
|
||||
MCInst const &ID = ISJ->getDesc();
|
||||
|
||||
if (HexagonMCInstrInfo::prefersSlot3(MCII, ID)) {
|
||||
++pSlot3Cnt;
|
||||
slot3ISJ = ISJ;
|
||||
}
|
||||
reservedSlots |= HexagonMCInstrInfo::getOtherReservedSlots(MCII, STI, ID);
|
||||
|
||||
switch (HexagonMCInstrInfo::getType(MCII, ID)) {
|
||||
case HexagonII::TypeS_2op:
|
||||
case HexagonII::TypeS_3op:
|
||||
case HexagonII::TypeALU64:
|
||||
break;
|
||||
case HexagonII::TypeJ:
|
||||
foundBranches.push_back(ISJ);
|
||||
break;
|
||||
case HexagonII::TypeCVI_VM_VP_LDU:
|
||||
case HexagonII::TypeCVI_VM_LD:
|
||||
case HexagonII::TypeCVI_VM_TMP_LD:
|
||||
case HexagonII::TypeCVI_GATHER:
|
||||
case HexagonII::TypeCVI_GATHER_RST:
|
||||
++NonZCVIloads;
|
||||
LLVM_FALLTHROUGH;
|
||||
case HexagonII::TypeCVI_ZW:
|
||||
++AllCVIloads;
|
||||
LLVM_FALLTHROUGH;
|
||||
case HexagonII::TypeLD:
|
||||
++loads;
|
||||
++memory;
|
||||
if (ISJ->Core.getUnits() == slotSingleLoad ||
|
||||
HexagonMCInstrInfo::getType(MCII, ID) == HexagonII::TypeCVI_VM_VP_LDU)
|
||||
++load0;
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, ID).isReturn())
|
||||
foundBranches.push_back(ISJ);
|
||||
break;
|
||||
case HexagonII::TypeCVI_VM_STU:
|
||||
case HexagonII::TypeCVI_VM_ST:
|
||||
case HexagonII::TypeCVI_VM_NEW_ST:
|
||||
case HexagonII::TypeCVI_SCATTER:
|
||||
case HexagonII::TypeCVI_SCATTER_DV:
|
||||
case HexagonII::TypeCVI_SCATTER_RST:
|
||||
case HexagonII::TypeCVI_SCATTER_NEW_RST:
|
||||
case HexagonII::TypeCVI_SCATTER_NEW_ST:
|
||||
++CVIstores;
|
||||
LLVM_FALLTHROUGH;
|
||||
case HexagonII::TypeST:
|
||||
++stores;
|
||||
++memory;
|
||||
if (ISJ->Core.getUnits() == slotSingleStore ||
|
||||
HexagonMCInstrInfo::getType(MCII, ID) == HexagonII::TypeCVI_VM_STU)
|
||||
++store0;
|
||||
break;
|
||||
case HexagonII::TypeV4LDST:
|
||||
++loads;
|
||||
++stores;
|
||||
++store1;
|
||||
++memops;
|
||||
++memory;
|
||||
break;
|
||||
case HexagonII::TypeNCJ:
|
||||
++memory; // NV insns are memory-like.
|
||||
foundBranches.push_back(ISJ);
|
||||
break;
|
||||
case HexagonII::TypeV2LDST:
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, ID).mayLoad()) {
|
||||
++loads;
|
||||
++memory;
|
||||
if (ISJ->Core.getUnits() == slotSingleLoad ||
|
||||
HexagonMCInstrInfo::getType(MCII, ID) ==
|
||||
HexagonII::TypeCVI_VM_VP_LDU)
|
||||
++load0;
|
||||
} else {
|
||||
assert(HexagonMCInstrInfo::getDesc(MCII, ID).mayStore());
|
||||
++memory;
|
||||
++stores;
|
||||
}
|
||||
break;
|
||||
case HexagonII::TypeCR:
|
||||
// Legacy conditional branch predicated on a register.
|
||||
case HexagonII::TypeCJ:
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, ID).isBranch())
|
||||
foundBranches.push_back(ISJ);
|
||||
break;
|
||||
case HexagonII::TypeDUPLEX: {
|
||||
++duplex;
|
||||
MCInst const &Inst0 = *ID.getOperand(0).getInst();
|
||||
MCInst const &Inst1 = *ID.getOperand(1).getInst();
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, Inst0).isBranch())
|
||||
foundBranches.push_back(ISJ);
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, Inst1).isBranch())
|
||||
foundBranches.push_back(ISJ);
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, Inst0).isReturn())
|
||||
foundBranches.push_back(ISJ);
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, Inst1).isReturn())
|
||||
foundBranches.push_back(ISJ);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
applySlotRestrictions();
|
||||
|
||||
// Check if the packet is legal.
|
||||
const unsigned ZCVIloads = AllCVIloads - NonZCVIloads;
|
||||
const bool ValidHVXMem =
|
||||
NonZCVIloads <= 1 && ZCVIloads <= 1 && CVIstores <= 1;
|
||||
if ((load0 > 1 || store0 > 1 || !ValidHVXMem) ||
|
||||
(duplex > 1 || (duplex && memory))) {
|
||||
reportError(llvm::Twine("invalid instruction packet"));
|
||||
return false;
|
||||
}
|
||||
|
||||
// Modify packet accordingly.
|
||||
// TODO: need to reserve slots #0 and #1 for duplex insns.
|
||||
bool bOnlySlot3 = false;
|
||||
for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
|
||||
MCInst const &ID = ISJ->getDesc();
|
||||
|
||||
if (!ISJ->Core.getUnits()) {
|
||||
// Error if insn may not be executed in any slot.
|
||||
return false;
|
||||
}
|
||||
|
||||
// A single load must use slot #0.
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, ID).mayLoad()) {
|
||||
if (loads == 1 && loads == memory && memops == 0)
|
||||
// Pin the load to slot #0.
|
||||
switch (ID.getOpcode()) {
|
||||
case Hexagon::V6_vgathermw:
|
||||
case Hexagon::V6_vgathermh:
|
||||
case Hexagon::V6_vgathermhw:
|
||||
case Hexagon::V6_vgathermwq:
|
||||
case Hexagon::V6_vgathermhq:
|
||||
case Hexagon::V6_vgathermhwq:
|
||||
// Slot1 only loads
|
||||
break;
|
||||
default:
|
||||
ISJ->Core.setUnits(ISJ->Core.getUnits() & slotSingleLoad);
|
||||
break;
|
||||
}
|
||||
else if (loads >= 1 && isMemReorderDisabled()) { // }:mem_noshuf
|
||||
// Loads must keep the original order ONLY if
|
||||
// isMemReorderDisabled() == true
|
||||
if (slotLoadStore < slotLastLoadStore) {
|
||||
// Error if no more slots available for loads.
|
||||
reportError(
|
||||
llvm::Twine("invalid instruction packet: too many loads"));
|
||||
return false;
|
||||
}
|
||||
// Pin the load to the highest slot available to it.
|
||||
ISJ->Core.setUnits(ISJ->Core.getUnits() & slotLoadStore);
|
||||
// Update the next highest slot available to loads.
|
||||
slotLoadStore >>= 1;
|
||||
}
|
||||
}
|
||||
|
||||
// A single store must use slot #0.
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, ID).mayStore()) {
|
||||
if (!store0) {
|
||||
if (stores == 1 && (loads == 0 || !isMemReorderDisabled()))
|
||||
// Pin the store to slot #0 only if isMemReorderDisabled() == false
|
||||
ISJ->Core.setUnits(ISJ->Core.getUnits() & slotSingleStore);
|
||||
else if (stores >= 1) {
|
||||
if (slotLoadStore < slotLastLoadStore) {
|
||||
// Error if no more slots available for stores.
|
||||
reportError(Twine("invalid instruction packet: too many stores"));
|
||||
return false;
|
||||
}
|
||||
// Pin the store to the highest slot available to it.
|
||||
ISJ->Core.setUnits(ISJ->Core.getUnits() & slotLoadStore);
|
||||
// Update the next highest slot available to stores.
|
||||
slotLoadStore >>= 1;
|
||||
}
|
||||
}
|
||||
if (store1 && stores > 1) {
|
||||
// Error if a single store with another store.
|
||||
reportError(Twine("invalid instruction packet: too many stores"));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// flag if an instruction requires to be in slot 3
|
||||
if (ISJ->Core.getUnits() == slotThree)
|
||||
bOnlySlot3 = true;
|
||||
|
||||
if (!ISJ->Core.getUnits()) {
|
||||
// Error if insn may not be executed in any slot.
|
||||
reportError(Twine("invalid instruction packet: out of slots"));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// These restrictions can modify the slot masks in the instructions
|
||||
// in the Packet member, but they can also detect constraint failures
|
||||
// which are fatal.
|
||||
if (!CheckFailure)
|
||||
restrictStoreLoadOrder(Summary);
|
||||
if (!CheckFailure)
|
||||
restrictBranchOrder(Summary);
|
||||
if (!CheckFailure)
|
||||
restrictPreferSlot3(Summary);
|
||||
return !CheckFailure;
|
||||
}
|
||||
|
||||
void HexagonShuffler::restrictBranchOrder(HexagonPacketSummary const &Summary) {
|
||||
// preserve branch order
|
||||
bool validateSlots = true;
|
||||
if (foundBranches.size() > 1) {
|
||||
if (foundBranches.size() > 2) {
|
||||
reportError(Twine("too many branches in packet"));
|
||||
return false;
|
||||
}
|
||||
const bool HasMultipleBranches = Summary.branchInsts.size() > 1;
|
||||
if (!HasMultipleBranches)
|
||||
return;
|
||||
|
||||
// try all possible choices
|
||||
for (unsigned int i = 0; i < MAX_JUMP_SLOTS; ++i) {
|
||||
// validate first jump with this slot rule
|
||||
if (!(jumpSlots[i].first & foundBranches[0]->Core.getUnits()))
|
||||
continue;
|
||||
|
||||
// validate second jump with this slot rule
|
||||
if (!(jumpSlots[i].second & foundBranches[1]->Core.getUnits()))
|
||||
continue;
|
||||
|
||||
// both valid for this configuration, set new slot rules
|
||||
PacketSave = Packet;
|
||||
foundBranches[0]->Core.setUnits(jumpSlots[i].first);
|
||||
foundBranches[1]->Core.setUnits(jumpSlots[i].second);
|
||||
|
||||
HexagonUnitAuction AuctionCore(reservedSlots);
|
||||
std::stable_sort(begin(), end(), HexagonInstr::lessCore);
|
||||
|
||||
// see if things ok with that instruction being pinned to slot "slotJump"
|
||||
bool bFail = false;
|
||||
for (iterator I = begin(); I != end() && !bFail; ++I)
|
||||
if (!AuctionCore.bid(I->Core.getUnits()))
|
||||
bFail = true;
|
||||
|
||||
// if yes, great, if not then restore original slot mask
|
||||
if (!bFail) {
|
||||
validateSlots = false; // all good, no need to re-do auction
|
||||
break;
|
||||
} else
|
||||
// restore original values
|
||||
Packet = PacketSave;
|
||||
}
|
||||
if (validateSlots) {
|
||||
reportError(Twine("invalid instruction packet: out of slots"));
|
||||
return false;
|
||||
}
|
||||
if (Summary.branchInsts.size() > 2) {
|
||||
reportError(Twine("too many branches in packet"));
|
||||
return;
|
||||
}
|
||||
|
||||
if (foundBranches.size() <= 1 && bOnlySlot3 == false && pSlot3Cnt == 1 &&
|
||||
slot3ISJ != end()) {
|
||||
validateSlots = true;
|
||||
// save off slot mask of instruction marked with A_PREFER_SLOT3
|
||||
// and then pin it to slot #3
|
||||
unsigned saveUnits = slot3ISJ->Core.getUnits();
|
||||
slot3ISJ->Core.setUnits(saveUnits & slotThree);
|
||||
// try all possible choices
|
||||
for (unsigned int i = 0; i < array_lengthof(jumpSlots); ++i) {
|
||||
// validate first jump with this slot rule
|
||||
if (!(jumpSlots[i].first & Summary.branchInsts[0]->Core.getUnits()))
|
||||
continue;
|
||||
|
||||
HexagonUnitAuction AuctionCore(reservedSlots);
|
||||
std::stable_sort(begin(), end(), HexagonInstr::lessCore);
|
||||
// validate second jump with this slot rule
|
||||
if (!(jumpSlots[i].second & Summary.branchInsts[1]->Core.getUnits()))
|
||||
continue;
|
||||
|
||||
// see if things ok with that instruction being pinned to slot #3
|
||||
bool bFail = false;
|
||||
for (iterator I = begin(); I != end() && !bFail; ++I)
|
||||
if (!AuctionCore.bid(I->Core.getUnits()))
|
||||
bFail = true;
|
||||
// both valid for this configuration, set new slot rules
|
||||
const HexagonPacket PacketSave = Packet;
|
||||
Summary.branchInsts[0]->Core.setUnits(jumpSlots[i].first);
|
||||
Summary.branchInsts[1]->Core.setUnits(jumpSlots[i].second);
|
||||
|
||||
const bool HasShuffledPacket = tryAuction(Summary).hasValue();
|
||||
if (HasShuffledPacket)
|
||||
return;
|
||||
|
||||
// if yes, great, if not then restore original slot mask
|
||||
if (!bFail)
|
||||
validateSlots = false; // all good, no need to re-do auction
|
||||
else
|
||||
for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
|
||||
MCInst const &ID = ISJ->getDesc();
|
||||
if (HexagonMCInstrInfo::prefersSlot3(MCII, ID))
|
||||
ISJ->Core.setUnits(saveUnits);
|
||||
}
|
||||
// restore original values
|
||||
Packet = PacketSave;
|
||||
}
|
||||
|
||||
// Check if any slot, core or CVI, is over-subscribed.
|
||||
// Verify the core slot subscriptions.
|
||||
if (validateSlots) {
|
||||
HexagonUnitAuction AuctionCore(reservedSlots);
|
||||
reportError("invalid instruction packet: out of slots");
|
||||
}
|
||||
|
||||
std::stable_sort(begin(), end(), HexagonInstr::lessCore);
|
||||
bool HexagonShuffler::ValidResourceUsage(HexagonPacketSummary const &Summary) {
|
||||
Optional<HexagonPacket> ShuffledPacket = tryAuction(Summary);
|
||||
|
||||
for (iterator I = begin(); I != end(); ++I)
|
||||
if (!AuctionCore.bid(I->Core.getUnits())) {
|
||||
reportError(Twine("invalid instruction packet: slot error"));
|
||||
return false;
|
||||
}
|
||||
if (!ShuffledPacket) {
|
||||
reportError("invalid instruction packet: slot error");
|
||||
return false;
|
||||
} else {
|
||||
Packet = *ShuffledPacket;
|
||||
}
|
||||
|
||||
// Verify the CVI slot subscriptions.
|
||||
std::stable_sort(begin(), end(), HexagonInstr::lessCVI);
|
||||
// create vector of hvx instructions to check
|
||||
HVXInstsT hvxInsts;
|
||||
hvxInsts.clear();
|
||||
for (iterator I = begin(); I != end(); ++I) {
|
||||
for (const_iterator I = cbegin(); I != cend(); ++I) {
|
||||
struct CVIUnits inst;
|
||||
inst.Units = I->CVI.getUnits();
|
||||
inst.Lanes = I->CVI.getLanes();
|
||||
@ -613,10 +372,286 @@ bool HexagonShuffler::check() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool HexagonShuffler::restrictStoreLoadOrder(
|
||||
HexagonPacketSummary const &Summary) {
|
||||
// Modify packet accordingly.
|
||||
// TODO: need to reserve slots #0 and #1 for duplex insns.
|
||||
static const unsigned slotFirstLoadStore = Slot1Mask;
|
||||
static const unsigned slotLastLoadStore = Slot0Mask;
|
||||
unsigned slotLoadStore = slotFirstLoadStore;
|
||||
|
||||
for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
|
||||
MCInst const &ID = ISJ->getDesc();
|
||||
|
||||
if (!ISJ->Core.getUnits())
|
||||
// Error if insn may not be executed in any slot.
|
||||
return false;
|
||||
|
||||
// A single load must use slot #0.
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, ID).mayLoad()) {
|
||||
if (Summary.loads == 1 && Summary.loads == Summary.memory &&
|
||||
Summary.memops == 0)
|
||||
// Pin the load to slot #0.
|
||||
switch (ID.getOpcode()) {
|
||||
case Hexagon::V6_vgathermw:
|
||||
case Hexagon::V6_vgathermh:
|
||||
case Hexagon::V6_vgathermhw:
|
||||
case Hexagon::V6_vgathermwq:
|
||||
case Hexagon::V6_vgathermhq:
|
||||
case Hexagon::V6_vgathermhwq:
|
||||
// Slot1 only loads
|
||||
break;
|
||||
default:
|
||||
ISJ->Core.setUnits(ISJ->Core.getUnits() & slotSingleLoad);
|
||||
break;
|
||||
}
|
||||
else if (Summary.loads >= 1 && isMemReorderDisabled()) { // }:mem_noshuf
|
||||
// Loads must keep the original order ONLY if
|
||||
// isMemReorderDisabled() == true
|
||||
if (slotLoadStore < slotLastLoadStore) {
|
||||
// Error if no more slots available for loads.
|
||||
reportError("invalid instruction packet: too many loads");
|
||||
return false;
|
||||
}
|
||||
// Pin the load to the highest slot available to it.
|
||||
ISJ->Core.setUnits(ISJ->Core.getUnits() & slotLoadStore);
|
||||
// Update the next highest slot available to loads.
|
||||
slotLoadStore >>= 1;
|
||||
}
|
||||
}
|
||||
|
||||
// A single store must use slot #0.
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, ID).mayStore()) {
|
||||
if (!Summary.store0) {
|
||||
if (Summary.stores == 1 &&
|
||||
(Summary.loads == 0 || !isMemReorderDisabled()))
|
||||
// Pin the store to slot #0 only if isMemReorderDisabled() == false
|
||||
ISJ->Core.setUnits(ISJ->Core.getUnits() & slotSingleStore);
|
||||
else if (Summary.stores >= 1) {
|
||||
if (slotLoadStore < slotLastLoadStore) {
|
||||
// Error if no more slots available for stores.
|
||||
reportError("invalid instruction packet: too many stores");
|
||||
return false;
|
||||
}
|
||||
// Pin the store to the highest slot available to it.
|
||||
ISJ->Core.setUnits(ISJ->Core.getUnits() & slotLoadStore);
|
||||
// Update the next highest slot available to stores.
|
||||
slotLoadStore >>= 1;
|
||||
}
|
||||
}
|
||||
if (Summary.store1 && Summary.stores > 1) {
|
||||
// Error if a single store with another store.
|
||||
reportError("invalid instruction packet: too many stores");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!ISJ->Core.getUnits()) {
|
||||
// Error if insn may not be executed in any slot.
|
||||
reportError("invalid instruction packet: out of slots");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
HexagonShuffler::HexagonPacketSummary HexagonShuffler::GetPacketSummary() {
|
||||
HexagonPacketSummary Summary = HexagonPacketSummary();
|
||||
|
||||
// Collect information from the insns in the packet.
|
||||
for (iterator ISJ = begin(); ISJ != end(); ++ISJ) {
|
||||
MCInst const &ID = ISJ->getDesc();
|
||||
|
||||
if (HexagonMCInstrInfo::isRestrictSlot1AOK(MCII, ID))
|
||||
Summary.Slot1AOKLoc = ID.getLoc();
|
||||
if (HexagonMCInstrInfo::isRestrictNoSlot1Store(MCII, ID))
|
||||
Summary.NoSlot1StoreLoc = ID.getLoc();
|
||||
|
||||
if (HexagonMCInstrInfo::prefersSlot3(MCII, ID)) {
|
||||
++Summary.pSlot3Cnt;
|
||||
Summary.PrefSlot3Inst = ISJ;
|
||||
}
|
||||
Summary.ReservedSlotMask |=
|
||||
HexagonMCInstrInfo::getOtherReservedSlots(MCII, STI, ID);
|
||||
|
||||
switch (HexagonMCInstrInfo::getType(MCII, ID)) {
|
||||
case HexagonII::TypeS_2op:
|
||||
case HexagonII::TypeS_3op:
|
||||
case HexagonII::TypeALU64:
|
||||
break;
|
||||
case HexagonII::TypeJ:
|
||||
Summary.branchInsts.push_back(ISJ);
|
||||
break;
|
||||
case HexagonII::TypeCVI_VM_VP_LDU:
|
||||
case HexagonII::TypeCVI_VM_LD:
|
||||
case HexagonII::TypeCVI_VM_TMP_LD:
|
||||
case HexagonII::TypeCVI_GATHER:
|
||||
case HexagonII::TypeCVI_GATHER_RST:
|
||||
++Summary.NonZCVIloads;
|
||||
LLVM_FALLTHROUGH;
|
||||
case HexagonII::TypeCVI_ZW:
|
||||
++Summary.AllCVIloads;
|
||||
LLVM_FALLTHROUGH;
|
||||
case HexagonII::TypeLD:
|
||||
++Summary.loads;
|
||||
++Summary.memory;
|
||||
if (ISJ->Core.getUnits() == slotSingleLoad ||
|
||||
HexagonMCInstrInfo::getType(MCII, ID) == HexagonII::TypeCVI_VM_VP_LDU)
|
||||
++Summary.load0;
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, ID).isReturn())
|
||||
Summary.branchInsts.push_back(ISJ);
|
||||
break;
|
||||
case HexagonII::TypeCVI_VM_STU:
|
||||
case HexagonII::TypeCVI_VM_ST:
|
||||
case HexagonII::TypeCVI_VM_NEW_ST:
|
||||
case HexagonII::TypeCVI_SCATTER:
|
||||
case HexagonII::TypeCVI_SCATTER_DV:
|
||||
case HexagonII::TypeCVI_SCATTER_RST:
|
||||
case HexagonII::TypeCVI_SCATTER_NEW_RST:
|
||||
case HexagonII::TypeCVI_SCATTER_NEW_ST:
|
||||
++Summary.CVIstores;
|
||||
LLVM_FALLTHROUGH;
|
||||
case HexagonII::TypeST:
|
||||
++Summary.stores;
|
||||
++Summary.memory;
|
||||
if (ISJ->Core.getUnits() == slotSingleStore ||
|
||||
HexagonMCInstrInfo::getType(MCII, ID) == HexagonII::TypeCVI_VM_STU)
|
||||
++Summary.store0;
|
||||
break;
|
||||
case HexagonII::TypeV4LDST:
|
||||
++Summary.loads;
|
||||
++Summary.stores;
|
||||
++Summary.store1;
|
||||
++Summary.memops;
|
||||
++Summary.memory;
|
||||
break;
|
||||
case HexagonII::TypeNCJ:
|
||||
++Summary.memory; // NV insns are memory-like.
|
||||
Summary.branchInsts.push_back(ISJ);
|
||||
break;
|
||||
case HexagonII::TypeV2LDST:
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, ID).mayLoad()) {
|
||||
++Summary.loads;
|
||||
++Summary.memory;
|
||||
if (ISJ->Core.getUnits() == slotSingleLoad ||
|
||||
HexagonMCInstrInfo::getType(MCII, ID) ==
|
||||
HexagonII::TypeCVI_VM_VP_LDU)
|
||||
++Summary.load0;
|
||||
} else {
|
||||
assert(HexagonMCInstrInfo::getDesc(MCII, ID).mayStore());
|
||||
++Summary.memory;
|
||||
++Summary.stores;
|
||||
}
|
||||
break;
|
||||
case HexagonII::TypeCR:
|
||||
// Legacy conditional branch predicated on a register.
|
||||
case HexagonII::TypeCJ:
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, ID).isBranch())
|
||||
Summary.branchInsts.push_back(ISJ);
|
||||
break;
|
||||
case HexagonII::TypeDUPLEX: {
|
||||
++Summary.duplex;
|
||||
MCInst const &Inst0 = *ID.getOperand(0).getInst();
|
||||
MCInst const &Inst1 = *ID.getOperand(1).getInst();
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, Inst0).isBranch())
|
||||
Summary.branchInsts.push_back(ISJ);
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, Inst1).isBranch())
|
||||
Summary.branchInsts.push_back(ISJ);
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, Inst0).isReturn())
|
||||
Summary.branchInsts.push_back(ISJ);
|
||||
if (HexagonMCInstrInfo::getDesc(MCII, Inst1).isReturn())
|
||||
Summary.branchInsts.push_back(ISJ);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return Summary;
|
||||
}
|
||||
|
||||
bool HexagonShuffler::ValidPacketMemoryOps(
|
||||
HexagonPacketSummary const &Summary) const {
|
||||
// Check if the packet is legal.
|
||||
const unsigned ZCVIloads = Summary.AllCVIloads - Summary.NonZCVIloads;
|
||||
const bool ValidHVXMem =
|
||||
Summary.NonZCVIloads <= 1 && ZCVIloads <= 1 && Summary.CVIstores <= 1;
|
||||
const bool InvalidPacket =
|
||||
((Summary.load0 > 1 || Summary.store0 > 1 || !ValidHVXMem) ||
|
||||
(Summary.duplex > 1 || (Summary.duplex && Summary.memory)));
|
||||
|
||||
return !InvalidPacket;
|
||||
}
|
||||
|
||||
void HexagonShuffler::restrictPreferSlot3(HexagonPacketSummary const &Summary) {
|
||||
// flag if an instruction requires to be in slot 3
|
||||
const bool HasOnlySlot3 = llvm::any_of(insts(), [&](HexagonInstr const &I) {
|
||||
return (I.Core.getUnits() == Slot3Mask);
|
||||
});
|
||||
const bool NeedsPrefSlot3Shuffle =
|
||||
(Summary.branchInsts.size() <= 1 && !HasOnlySlot3 &&
|
||||
Summary.pSlot3Cnt == 1 && Summary.PrefSlot3Inst);
|
||||
|
||||
if (!NeedsPrefSlot3Shuffle)
|
||||
return;
|
||||
|
||||
HexagonInstr *PrefSlot3Inst = *Summary.PrefSlot3Inst;
|
||||
// save off slot mask of instruction marked with A_PREFER_SLOT3
|
||||
// and then pin it to slot #3
|
||||
const unsigned saveUnits = PrefSlot3Inst->Core.getUnits();
|
||||
PrefSlot3Inst->Core.setUnits(saveUnits & Slot3Mask);
|
||||
const bool HasShuffledPacket = tryAuction(Summary).hasValue();
|
||||
if (HasShuffledPacket)
|
||||
return;
|
||||
|
||||
PrefSlot3Inst->Core.setUnits(saveUnits);
|
||||
}
|
||||
|
||||
/// Check that the packet is legal and enforce relative insn order.
|
||||
bool HexagonShuffler::check() {
|
||||
const HexagonPacketSummary Summary = GetPacketSummary();
|
||||
if (!applySlotRestrictions(Summary))
|
||||
return false;
|
||||
|
||||
if (!ValidPacketMemoryOps(Summary)) {
|
||||
reportError("invalid instruction packet");
|
||||
return false;
|
||||
}
|
||||
|
||||
ValidResourceUsage(Summary);
|
||||
|
||||
return !CheckFailure;
|
||||
}
|
||||
|
||||
llvm::Optional<HexagonShuffler::HexagonPacket>
|
||||
HexagonShuffler::tryAuction(HexagonPacketSummary const &Summary) const {
|
||||
HexagonPacket PacketResult = Packet;
|
||||
HexagonUnitAuction AuctionCore(Summary.ReservedSlotMask);
|
||||
std::stable_sort(PacketResult.begin(), PacketResult.end(),
|
||||
HexagonInstr::lessCore);
|
||||
|
||||
const bool ValidSlots =
|
||||
llvm::all_of(insts(PacketResult), [&AuctionCore](HexagonInstr const &I) {
|
||||
return AuctionCore.bid(I.Core.getUnits());
|
||||
});
|
||||
|
||||
LLVM_DEBUG(
|
||||
dbgs() << "Shuffle attempt: " << (ValidSlots ? "passed" : "failed")
|
||||
<< "\n";
|
||||
for (HexagonInstr const &ISJ : insts(PacketResult))
|
||||
dbgs() << "\t" << HexagonMCInstrInfo::getName(MCII, *ISJ.ID) << ": "
|
||||
<< llvm::format_hex(ISJ.Core.getUnits(), 4, true) << "\n";
|
||||
);
|
||||
|
||||
Optional<HexagonPacket> Res;
|
||||
if (ValidSlots)
|
||||
Res = PacketResult;
|
||||
|
||||
return Res;
|
||||
}
|
||||
|
||||
bool HexagonShuffler::shuffle() {
|
||||
if (size() > HEXAGON_PACKET_SIZE) {
|
||||
// Ignore a packet with with more than what a packet can hold
|
||||
@ -653,20 +688,25 @@ bool HexagonShuffler::shuffle() {
|
||||
++emptySlots;
|
||||
}
|
||||
|
||||
for (iterator ISJ = begin(); ISJ != end(); ++ISJ)
|
||||
LLVM_DEBUG(dbgs().write_hex(ISJ->Core.getUnits()); if (ISJ->CVI.isValid()) {
|
||||
dbgs() << '/';
|
||||
dbgs().write_hex(ISJ->CVI.getUnits()) << '|';
|
||||
dbgs() << ISJ->CVI.getLanes();
|
||||
} dbgs() << ':'
|
||||
<< HexagonMCInstrInfo::getDesc(MCII, ISJ->getDesc()).getOpcode();
|
||||
dbgs() << '\n');
|
||||
LLVM_DEBUG(dbgs() << '\n');
|
||||
LLVM_DEBUG(
|
||||
for (HexagonInstr const &ISJ : insts()) {
|
||||
dbgs().write_hex(ISJ.Core.getUnits());
|
||||
if (ISJ.CVI.isValid()) {
|
||||
dbgs() << '/';
|
||||
dbgs().write_hex(ISJ.CVI.getUnits()) << '|';
|
||||
dbgs() << ISJ.CVI.getLanes();
|
||||
}
|
||||
dbgs() << ':'
|
||||
<< HexagonMCInstrInfo::getDesc(MCII, ISJ.getDesc()).getOpcode()
|
||||
<< '\n';
|
||||
} dbgs() << '\n';
|
||||
);
|
||||
|
||||
return Ok;
|
||||
}
|
||||
|
||||
void HexagonShuffler::reportError(Twine const &Msg) {
|
||||
CheckFailure = true;
|
||||
if (ReportErrors) {
|
||||
for (auto const &I : AppliedRestrictions) {
|
||||
auto SM = Context.getSourceManager();
|
||||
|
||||
@ -17,11 +17,13 @@
|
||||
#include "MCTargetDesc/HexagonMCInstrInfo.h"
|
||||
#include "MCTargetDesc/HexagonMCTargetDesc.h"
|
||||
#include "llvm/ADT/DenseMap.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
#include "llvm/Support/MathExtras.h"
|
||||
#include "llvm/Support/SMLoc.h"
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <utility>
|
||||
|
||||
namespace llvm {
|
||||
@ -140,9 +142,30 @@ class HexagonShuffler {
|
||||
using HexagonPacket =
|
||||
SmallVector<HexagonInstr, HEXAGON_PRESHUFFLE_PACKET_SIZE>;
|
||||
|
||||
struct HexagonPacketSummary {
|
||||
// Number of memory operations, loads, solo loads, stores, solo stores,
|
||||
// single stores.
|
||||
unsigned memory;
|
||||
unsigned loads;
|
||||
unsigned load0;
|
||||
unsigned stores;
|
||||
unsigned store0;
|
||||
unsigned store1;
|
||||
unsigned NonZCVIloads;
|
||||
unsigned AllCVIloads;
|
||||
unsigned CVIstores;
|
||||
// Number of duplex insns
|
||||
unsigned duplex;
|
||||
unsigned pSlot3Cnt;
|
||||
Optional<HexagonInstr *> PrefSlot3Inst;
|
||||
unsigned memops;
|
||||
unsigned ReservedSlotMask;
|
||||
SmallVector<HexagonInstr *, HEXAGON_PRESHUFFLE_PACKET_SIZE> branchInsts;
|
||||
Optional<SMLoc> Slot1AOKLoc;
|
||||
Optional<SMLoc> NoSlot1StoreLoc;
|
||||
};
|
||||
// Insn handles in a bundle.
|
||||
HexagonPacket Packet;
|
||||
HexagonPacket PacketSave;
|
||||
|
||||
HexagonCVIResource::TypeUnitsAndLanes TUL;
|
||||
|
||||
@ -153,13 +176,28 @@ protected:
|
||||
MCSubtargetInfo const &STI;
|
||||
SMLoc Loc;
|
||||
bool ReportErrors;
|
||||
bool CheckFailure;
|
||||
std::vector<std::pair<SMLoc, std::string>> AppliedRestrictions;
|
||||
void applySlotRestrictions();
|
||||
void restrictSlot1AOK();
|
||||
void restrictNoSlot1Store();
|
||||
bool applySlotRestrictions(HexagonPacketSummary const &Summary);
|
||||
void restrictSlot1AOK(HexagonPacketSummary const &Summary);
|
||||
void restrictNoSlot1Store(HexagonPacketSummary const &Summary);
|
||||
void restrictNoSlot1();
|
||||
|
||||
Optional<HexagonPacket> tryAuction(HexagonPacketSummary const &Summary) const;
|
||||
|
||||
bool restrictStoreLoadOrder(HexagonPacketSummary const &Summary);
|
||||
void restrictBranchOrder(HexagonPacketSummary const &Summary);
|
||||
void restrictPreferSlot3(HexagonPacketSummary const &Summary);
|
||||
HexagonPacketSummary GetPacketSummary();
|
||||
bool ValidPacketMemoryOps(HexagonPacketSummary const &Summary) const;
|
||||
bool ValidResourceUsage(HexagonPacketSummary const &Summary);
|
||||
bool validPacketInsts() const;
|
||||
|
||||
public:
|
||||
using iterator = HexagonPacket::iterator;
|
||||
using const_iterator = HexagonPacket::const_iterator;
|
||||
using packet_range = iterator_range<HexagonPacket::iterator>;
|
||||
using const_packet_range = iterator_range<HexagonPacket::const_iterator>;
|
||||
|
||||
HexagonShuffler(MCContext &Context, bool ReportErrors,
|
||||
MCInstrInfo const &MCII, MCSubtargetInfo const &STI);
|
||||
@ -179,6 +217,26 @@ public:
|
||||
|
||||
iterator begin() { return (Packet.begin()); }
|
||||
iterator end() { return (Packet.end()); }
|
||||
const_iterator cbegin() const { return (Packet.begin()); }
|
||||
const_iterator cend() const { return (Packet.end()); }
|
||||
packet_range insts(HexagonPacket &P) {
|
||||
return make_range(P.begin(), P.end());
|
||||
}
|
||||
const_packet_range insts(HexagonPacket const &P) const {
|
||||
return make_range(P.begin(), P.end());
|
||||
}
|
||||
packet_range insts() { return make_range(begin(), end()); }
|
||||
const_packet_range insts() const { return make_range(cbegin(), cend()); }
|
||||
|
||||
using InstPredicate = bool (*)(MCInstrInfo const &, MCInst const &);
|
||||
|
||||
bool HasInstWith(InstPredicate Pred) const {
|
||||
return llvm::any_of(make_range(cbegin(), cend()),
|
||||
[&](HexagonInstr const &I) {
|
||||
MCInst const &Inst = I.getDesc();
|
||||
return (*Pred)(MCII, Inst);
|
||||
});
|
||||
}
|
||||
|
||||
// Add insn handle to the bundle .
|
||||
void append(MCInst const &ID, MCInst const *Extender, unsigned S);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user