From 28262fbaf29ab1f28519e0d398a39e6deded1289 Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Sat, 24 Nov 2007 07:07:01 +0000 Subject: [PATCH] Several changes: 1) Change the interface to TargetLowering::ExpandOperationResult to take and return entire NODES that need a result expanded, not just the value. This allows us to handle things like READCYCLECOUNTER, which returns two values. 2) Implement (extremely limited) support in LegalizeDAG::ExpandOp for MERGE_VALUES. 3) Reimplement custom lowering in LegalizeDAGTypes in terms of the new ExpandOperationResult. This makes the result simpler and fully general. 4) Implement (fully general) expand support for MERGE_VALUES in LegalizeDAGTypes. 5) Implement ExpandOperationResult support for ARM f64->i64 bitconvert and ARM i64 shifts, allowing them to work with LegalizeDAGTypes. 6) Implement ExpandOperationResult support for X86 READCYCLECOUNTER and FP_TO_SINT, allowing them to work with LegalizeDAGTypes. LegalizeDAGTypes now passes several more X86 codegen tests when enabled and when type legalization in LegalizeDAG is ifdef'd out. llvm-svn: 44300 --- include/llvm/Target/TargetLowering.h | 10 +- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 40 ++++-- lib/CodeGen/SelectionDAG/LegalizeDAGTypes.cpp | 113 +++++++++++---- lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 10 +- lib/Target/ARM/ARMISelLowering.cpp | 130 ++++++++++-------- lib/Target/ARM/ARMISelLowering.h | 2 + lib/Target/X86/X86ISelLowering.cpp | 111 +++++++++------ lib/Target/X86/X86ISelLowering.h | 12 +- 8 files changed, 278 insertions(+), 150 deletions(-) diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h index ee265791168..5439ff667f4 100644 --- a/include/llvm/Target/TargetLowering.h +++ b/include/llvm/Target/TargetLowering.h @@ -910,12 +910,16 @@ public: /// ExpandOperationResult - This callback is invoked for operations that are /// unsupported by the target, which are registered to use 'custom' lowering, - /// and whose result type needs to be expanded. + /// and whose result type needs to be expanded. This must return a node whose + /// results precisely match the results of the input node. This typically + /// involves a MERGE_VALUES node and/or BUILD_PAIR. /// /// If the target has no operations that require custom lowering, it need not /// implement this. The default implementation of this aborts. - virtual std::pair - ExpandOperationResult(SDNode *N, SelectionDAG &DAG); + virtual SDNode *ExpandOperationResult(SDNode *N, SelectionDAG &DAG) { + assert(0 && "ExpandOperationResult not implemented for this target!"); + return 0; + } /// IsEligibleForTailCallOptimization - Check whether the call is eligible for /// tail call optimization. Targets which want to do tail call optimization diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 32af553b9bf..aa8ffaae071 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -5187,10 +5187,21 @@ SDOperand SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDOperand LegalOp, // Otherwise, try a larger type. } - // Okay, we found the operation and type to use. Truncate the result of the - // extended FP_TO_*INT operation to the desired size. - return DAG.getNode(ISD::TRUNCATE, DestVT, - DAG.getNode(OpToUse, NewOutTy, LegalOp)); + + // Okay, we found the operation and type to use. + SDOperand Operation = DAG.getNode(OpToUse, NewOutTy, LegalOp); + + // If the operation produces an invalid type, it must be custom lowered. Use + // the target lowering hooks to expand it. Just keep the low part of the + // expanded operation, we know that we're truncating anyway. + if (getTypeAction(NewOutTy) == Expand) { + Operation = SDOperand(TLI.ExpandOperationResult(Operation.Val, DAG), 0); + assert(Operation.Val && "Didn't return anything"); + } + + // Truncate the result of the extended FP_TO_*INT operation to the desired + // size. + return DAG.getNode(ISD::TRUNCATE, DestVT, Operation); } /// ExpandBSWAP - Open code the operations for BSWAP of the specified operation. @@ -5388,6 +5399,16 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ Lo = Node->getOperand(0); Hi = Node->getOperand(1); break; + + case ISD::MERGE_VALUES: + // FIXME: For now only expand i64,chain = MERGE_VALUES (x, y) + assert(Op.ResNo == 0 && Node->getNumValues() == 2 && + Op.getValue(1).getValueType() == MVT::Other && + "unhandled MERGE_VALUES"); + ExpandOp(Op.getOperand(0), Lo, Hi); + // Remember that we legalized the chain. + AddLegalizedOperand(Op.getValue(1), LegalizeOp(Op.getOperand(1))); + break; case ISD::SIGN_EXTEND_INREG: ExpandOp(Node->getOperand(0), Lo, Hi); @@ -5652,16 +5673,17 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ break; } - case ISD::READCYCLECOUNTER: + case ISD::READCYCLECOUNTER: { assert(TLI.getOperationAction(ISD::READCYCLECOUNTER, VT) == TargetLowering::Custom && "Must custom expand ReadCycleCounter"); - Lo = TLI.LowerOperation(Op, DAG); - assert(Lo.Val && "Node must be custom expanded!"); - Hi = Lo.getValue(1); + SDOperand Tmp = TLI.LowerOperation(Op, DAG); + assert(Tmp.Val && "Node must be custom expanded!"); + ExpandOp(Tmp.getValue(0), Lo, Hi); AddLegalizedOperand(SDOperand(Node, 1), // Remember we legalized the chain. - LegalizeOp(Lo.getValue(2))); + LegalizeOp(Tmp.getValue(1))); break; + } // These operators cannot be expanded directly, emit them as calls to // library functions. diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAGTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAGTypes.cpp index bb65a672041..5be8253c86b 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAGTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAGTypes.cpp @@ -115,7 +115,8 @@ public: private: void MarkNewNodes(SDNode *N); - void ReplaceLegalValueWith(SDOperand From, SDOperand To); + void ReplaceValueWith(SDOperand From, SDOperand To); + void ReplaceNodeWith(SDNode *From, SDNode *To); void RemapNode(SDOperand &N); @@ -167,6 +168,7 @@ private: void ExpandResult_UNDEF (SDNode *N, SDOperand &Lo, SDOperand &Hi); void ExpandResult_Constant (SDNode *N, SDOperand &Lo, SDOperand &Hi); void ExpandResult_BUILD_PAIR (SDNode *N, SDOperand &Lo, SDOperand &Hi); + void ExpandResult_MERGE_VALUES(SDNode *N, SDOperand &Lo, SDOperand &Hi); void ExpandResult_ANY_EXTEND (SDNode *N, SDOperand &Lo, SDOperand &Hi); void ExpandResult_ZERO_EXTEND(SDNode *N, SDOperand &Lo, SDOperand &Hi); void ExpandResult_SIGN_EXTEND(SDNode *N, SDOperand &Lo, SDOperand &Hi); @@ -391,10 +393,10 @@ void DAGTypeLegalizer::MarkNewNodes(SDNode *N) { Worklist.push_back(N); } -/// ReplaceLegalValueWith - The specified value with a legal type was legalized -/// to the specified other value. If they are different, update the DAG and -/// NodeIDs replacing any uses of From to use To instead. -void DAGTypeLegalizer::ReplaceLegalValueWith(SDOperand From, SDOperand To) { +/// ReplaceValueWith - The specified value was legalized to the specified other +/// value. If they are different, update the DAG and NodeIDs replacing any uses +/// of From to use To instead. +void DAGTypeLegalizer::ReplaceValueWith(SDOperand From, SDOperand To) { if (From == To) return; // If expansion produced new nodes, make sure they are properly marked. @@ -410,8 +412,8 @@ void DAGTypeLegalizer::ReplaceLegalValueWith(SDOperand From, SDOperand To) { ReplacedNodes[From] = To; // Since we just made an unstructured update to the DAG, which could wreak - // general havoc on anything that once used N and now uses Res, walk all users - // of the result, updating their flags. + // general havoc on anything that once used From and now uses To, walk all + // users of the result, updating their flags. for (SDNode::use_iterator I = To.Val->use_begin(), E = To.Val->use_end(); I != E; ++I) { SDNode *User = *I; @@ -425,14 +427,51 @@ void DAGTypeLegalizer::ReplaceLegalValueWith(SDOperand From, SDOperand To) { } } +/// ReplaceNodeWith - Replace uses of the 'from' node's results with the 'to' +/// node's results. The from and to node must define identical result types. +void DAGTypeLegalizer::ReplaceNodeWith(SDNode *From, SDNode *To) { + if (From == To) return; + assert(From->getNumValues() == To->getNumValues() && + "Node results don't match"); + + // If expansion produced new nodes, make sure they are properly marked. + if (To->getNodeId() == NewNode) + MarkNewNodes(To); + + // Anything that used the old node should now use the new one. Note that this + // can potentially cause recursive merging. + DAG.ReplaceAllUsesWith(From, To); + + // The old node may still be present in ExpandedNodes or PromotedNodes. + // Inform them about the replacement. + for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) { + assert(From->getValueType(i) == To->getValueType(i) && + "Node results don't match"); + ReplacedNodes[SDOperand(From, i)] = SDOperand(To, i); + } + + // Since we just made an unstructured update to the DAG, which could wreak + // general havoc on anything that once used From and now uses To, walk all + // users of the result, updating their flags. + for (SDNode::use_iterator I = To->use_begin(), E = To->use_end();I != E; ++I){ + SDNode *User = *I; + // If the node isn't already processed or in the worklist, mark it as new, + // then use MarkNewNodes to recompute its ID. + int NodeId = User->getNodeId(); + if (NodeId != ReadyToProcess && NodeId != Processed) { + User->setNodeId(NewNode); + MarkNewNodes(User); + } + } +} + + /// RemapNode - If the specified value was already legalized to another value, /// replace it by that value. void DAGTypeLegalizer::RemapNode(SDOperand &N) { - DenseMap::iterator I = ReplacedNodes.find(N); - if (I != ReplacedNodes.end()) { - RemapNode(I->second); + for (DenseMap::iterator I = ReplacedNodes.find(N); + I != ReplacedNodes.end(); I = ReplacedNodes.find(N)) N = I->second; - } } void DAGTypeLegalizer::SetPromotedOp(SDOperand Op, SDOperand Result) { @@ -712,7 +751,7 @@ SDOperand DAGTypeLegalizer::PromoteResult_LOAD(LoadSDNode *N) { // Legalized the chain result - switch anything that used the old chain to // use the new one. - ReplaceLegalValueWith(SDOperand(N, 1), Res.getValue(1)); + ReplaceValueWith(SDOperand(N, 1), Res.getValue(1)); return Res; } @@ -798,15 +837,14 @@ void DAGTypeLegalizer::ExpandResult(SDNode *N, unsigned ResNo) { SDOperand Lo, Hi; Lo = Hi = SDOperand(); - // If this is a single-result node, see if the target wants to custom expand - // it. - if (N->getNumValues() == 1 && - TLI.getOperationAction(N->getOpcode(), - N->getValueType(0)) == TargetLowering::Custom) { + // See if the target wants to custom expand this node. + if (TLI.getOperationAction(N->getOpcode(), N->getValueType(0)) == + TargetLowering::Custom) { // If the target wants to, allow it to lower this itself. - std::pair P = TLI.ExpandOperationResult(N, DAG); - if (P.first.Val) { - SetExpandedOp(SDOperand(N, ResNo), P.first, P.second); + if (SDNode *P = TLI.ExpandOperationResult(N, DAG)) { + // Everything that once used N now uses P. P had better not require + // custom expansion. + ReplaceNodeWith(N, P); return; } } @@ -817,12 +855,13 @@ void DAGTypeLegalizer::ExpandResult(SDNode *N, unsigned ResNo) { cerr << "ExpandResult #" << ResNo << ": "; N->dump(&DAG); cerr << "\n"; #endif - assert(0 && "Do not know how to expand this operator!"); + assert(0 && "Do not know how to expand the result of this operator!"); abort(); case ISD::UNDEF: ExpandResult_UNDEF(N, Lo, Hi); break; case ISD::Constant: ExpandResult_Constant(N, Lo, Hi); break; case ISD::BUILD_PAIR: ExpandResult_BUILD_PAIR(N, Lo, Hi); break; + case ISD::MERGE_VALUES: ExpandResult_MERGE_VALUES(N, Lo, Hi); break; case ISD::ANY_EXTEND: ExpandResult_ANY_EXTEND(N, Lo, Hi); break; case ISD::ZERO_EXTEND: ExpandResult_ZERO_EXTEND(N, Lo, Hi); break; case ISD::SIGN_EXTEND: ExpandResult_SIGN_EXTEND(N, Lo, Hi); break; @@ -846,7 +885,6 @@ void DAGTypeLegalizer::ExpandResult(SDNode *N, unsigned ResNo) { case ISD::SHL: case ISD::SRA: case ISD::SRL: ExpandResult_Shift(N, Lo, Hi); break; - } // If Lo/Hi is null, the sub-method took care of registering results etc. @@ -875,6 +913,27 @@ void DAGTypeLegalizer::ExpandResult_BUILD_PAIR(SDNode *N, Hi = N->getOperand(1); } +void DAGTypeLegalizer::ExpandResult_MERGE_VALUES(SDNode *N, + SDOperand &Lo, SDOperand &Hi) { + // A MERGE_VALUES node can produce any number of values. We know that the + // first illegal one needs to be expanded into Lo/Hi. + unsigned i; + + // The string of legal results gets turns into the input operands, which have + // the same type. + for (i = 0; isTypeLegal(N->getValueType(i)); ++i) + ReplaceValueWith(SDOperand(N, i), SDOperand(N->getOperand(i))); + + // The first illegal result must be the one that needs to be expanded. + GetExpandedOp(N->getOperand(i), Lo, Hi); + + // Legalize the rest of the results into the input operands whether they are + // legal or not. + unsigned e = N->getNumValues(); + for (++i; i != e; ++i) + ReplaceValueWith(SDOperand(N, i), SDOperand(N->getOperand(i))); +} + void DAGTypeLegalizer::ExpandResult_ANY_EXTEND(SDNode *N, SDOperand &Lo, SDOperand &Hi) { MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0)); @@ -1096,7 +1155,7 @@ void DAGTypeLegalizer::ExpandResult_LOAD(LoadSDNode *N, // Legalized the chain result - switch anything that used the old chain to // use the new one. - ReplaceLegalValueWith(SDOperand(N, 1), Ch); + ReplaceValueWith(SDOperand(N, 1), Ch); } void DAGTypeLegalizer::ExpandResult_Logical(SDNode *N, @@ -1184,7 +1243,7 @@ void DAGTypeLegalizer::ExpandResult_ADDSUBC(SDNode *N, // Legalized the flag result - switch anything that used the old flag to // use the new one. - ReplaceLegalValueWith(SDOperand(N, 1), Hi.getValue(1)); + ReplaceValueWith(SDOperand(N, 1), Hi.getValue(1)); } void DAGTypeLegalizer::ExpandResult_ADDSUBE(SDNode *N, @@ -1203,7 +1262,7 @@ void DAGTypeLegalizer::ExpandResult_ADDSUBE(SDNode *N, // Legalized the flag result - switch anything that used the old flag to // use the new one. - ReplaceLegalValueWith(SDOperand(N, 1), Hi.getValue(1)); + ReplaceValueWith(SDOperand(N, 1), Hi.getValue(1)); } void DAGTypeLegalizer::ExpandResult_MUL(SDNode *N, @@ -1537,7 +1596,7 @@ bool DAGTypeLegalizer::PromoteOperand(SDNode *N, unsigned OpNo) { assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && "Invalid operand expansion"); - ReplaceLegalValueWith(SDOperand(N, 0), Res); + ReplaceValueWith(SDOperand(N, 0), Res); return false; } @@ -1759,7 +1818,7 @@ bool DAGTypeLegalizer::ExpandOperand(SDNode *N, unsigned OpNo) { assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && "Invalid operand expansion"); - ReplaceLegalValueWith(SDOperand(N, 0), Res); + ReplaceValueWith(SDOperand(N, 0), Res); return false; } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index ade0a2860b5..270c9a958f2 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -1921,7 +1921,7 @@ unsigned SelectionDAGLowering::Clusterify(CaseVector& Cases, if (Cases.size()>=2) // Must recompute end() each iteration because it may be // invalidated by erase if we hold on to it - for (CaseItr I=Cases.begin(), J=next(Cases.begin()); J!=Cases.end(); ) { + for (CaseItr I=Cases.begin(), J=++(Cases.begin()); J!=Cases.end(); ) { int64_t nextValue = cast(J->Low)->getSExtValue(); int64_t currentValue = cast(I->High)->getSExtValue(); MachineBasicBlock* nextBB = J->BB; @@ -4144,14 +4144,6 @@ SDOperand TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { return SDOperand(); } -std::pair -TargetLowering::ExpandOperationResult(SDNode *N, SelectionDAG &DAG) { - assert(0 && "ExpandOperation not implemented for this target!"); - abort(); - return std::pair(); -} - - SDOperand TargetLowering::CustomPromoteOperation(SDOperand Op, SelectionDAG &DAG) { assert(0 && "CustomPromoteOperation not implemented for this target!"); diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp index d1cd8c0c636..608cc4c15de 100644 --- a/lib/Target/ARM/ARMISelLowering.cpp +++ b/lib/Target/ARM/ARMISelLowering.cpp @@ -949,10 +949,8 @@ static SDOperand LowerFORMAL_ARGUMENT(SDOperand Op, SelectionDAG &DAG, vRegs[NumGPRs+1] = VReg; SDOperand ArgValue2 = DAG.getCopyFromReg(Root, VReg, MVT::i32); - if (ObjectVT == MVT::i64) - ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, ArgValue, ArgValue2); - else - ArgValue = DAG.getNode(ARMISD::FMDRR, MVT::f64, ArgValue, ArgValue2); + assert(ObjectVT != MVT::i64 && "i64 should already be lowered"); + ArgValue = DAG.getNode(ARMISD::FMDRR, MVT::f64, ArgValue, ArgValue2); } NumGPRs += ObjGPRs; @@ -966,12 +964,9 @@ static SDOperand LowerFORMAL_ARGUMENT(SDOperand Op, SelectionDAG &DAG, if (ObjGPRs == 0) ArgValue = DAG.getLoad(ObjectVT, Root, FIN, NULL, 0); else { - SDOperand ArgValue2 = - DAG.getLoad(MVT::i32, Root, FIN, NULL, 0); - if (ObjectVT == MVT::i64) - ArgValue= DAG.getNode(ISD::BUILD_PAIR, MVT::i64, ArgValue, ArgValue2); - else - ArgValue= DAG.getNode(ARMISD::FMDRR, MVT::f64, ArgValue, ArgValue2); + SDOperand ArgValue2 = DAG.getLoad(MVT::i32, Root, FIN, NULL, 0); + assert(ObjectVT != MVT::i64 && "i64 should already be lowered"); + ArgValue = DAG.getNode(ARMISD::FMDRR, MVT::f64, ArgValue, ArgValue2); } } else { // Don't emit a dead load. @@ -1256,51 +1251,6 @@ static SDOperand LowerFCOPYSIGN(SDOperand Op, SelectionDAG &DAG) { return DAG.getNode(ARMISD::CNEG, VT, AbsVal, AbsVal, ARMCC, CCR, Cmp); } -static SDOperand LowerBIT_CONVERT(SDOperand Op, SelectionDAG &DAG) { - // Turn f64->i64 into FMRRD. - assert(Op.getValueType() == MVT::i64 && - Op.getOperand(0).getValueType() == MVT::f64); - - Op = Op.getOperand(0); - SDOperand Cvt = DAG.getNode(ARMISD::FMRRD, DAG.getVTList(MVT::i32, MVT::i32), - &Op, 1); - - // Merge the pieces into a single i64 value. - return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Cvt, Cvt.getValue(1)); -} - -static SDOperand LowerSRx(SDOperand Op, SelectionDAG &DAG, - const ARMSubtarget *ST) { - assert(Op.getValueType() == MVT::i64 && - (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) && - "Unknown shift to lower!"); - - // We only lower SRA, SRL of 1 here, all others use generic lowering. - if (!isa(Op.getOperand(1)) || - cast(Op.getOperand(1))->getValue() != 1) - return SDOperand(); - - // If we are in thumb mode, we don't have RRX. - if (ST->isThumb()) return SDOperand(); - - // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr. - SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op.getOperand(0), - DAG.getConstant(0, MVT::i32)); - SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op.getOperand(0), - DAG.getConstant(1, MVT::i32)); - - // First, build a SRA_FLAG/SRL_FLAG op, which shifts the top part by one and - // captures the result into a carry flag. - unsigned Opc = Op.getOpcode() == ISD::SRL ? ARMISD::SRL_FLAG:ARMISD::SRA_FLAG; - Hi = DAG.getNode(Opc, DAG.getVTList(MVT::i32, MVT::Flag), &Hi, 1); - - // The low part is an ARMISD::RRX operand, which shifts the carry in. - Lo = DAG.getNode(ARMISD::RRX, MVT::i32, Lo, Hi.getValue(1)); - - // Merge the pieces into a single i64 value. - return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi); -} - SDOperand ARMTargetLowering::LowerMEMCPYInline(SDOperand Chain, SDOperand Dest, SDOperand Source, @@ -1396,6 +1346,51 @@ SDOperand ARMTargetLowering::LowerMEMCPYInline(SDOperand Chain, return DAG.getNode(ISD::TokenFactor, MVT::Other, &TFOps[0], i); } +static SDNode *ExpandBIT_CONVERT(SDNode *N, SelectionDAG &DAG) { + // Turn f64->i64 into FMRRD. + assert(N->getValueType(0) == MVT::i64 && + N->getOperand(0).getValueType() == MVT::f64); + + SDOperand Op = N->getOperand(0); + SDOperand Cvt = DAG.getNode(ARMISD::FMRRD, DAG.getVTList(MVT::i32, MVT::i32), + &Op, 1); + + // Merge the pieces into a single i64 value. + return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Cvt, Cvt.getValue(1)).Val; +} + +static SDNode *ExpandSRx(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST) { + assert(N->getValueType(0) == MVT::i64 && + (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) && + "Unknown shift to lower!"); + + // We only lower SRA, SRL of 1 here, all others use generic lowering. + if (!isa(N->getOperand(1)) || + cast(N->getOperand(1))->getValue() != 1) + return 0; + + // If we are in thumb mode, we don't have RRX. + if (ST->isThumb()) return 0; + + // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr. + SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, N->getOperand(0), + DAG.getConstant(0, MVT::i32)); + SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, N->getOperand(0), + DAG.getConstant(1, MVT::i32)); + + // First, build a SRA_FLAG/SRL_FLAG op, which shifts the top part by one and + // captures the result into a carry flag. + unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::SRL_FLAG:ARMISD::SRA_FLAG; + Hi = DAG.getNode(Opc, DAG.getVTList(MVT::i32, MVT::Flag), &Hi, 1); + + // The low part is an ARMISD::RRX operand, which shifts the carry in. + Lo = DAG.getNode(ARMISD::RRX, MVT::i32, Lo, Hi.getValue(1)); + + // Merge the pieces into a single i64 value. + return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi).Val; +} + + SDOperand ARMTargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { switch (Op.getOpcode()) { default: assert(0 && "Don't know how to custom lower this!"); abort(); @@ -1415,20 +1410,35 @@ SDOperand ARMTargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG); case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); - case ISD::BIT_CONVERT: return LowerBIT_CONVERT(Op, DAG); - case ISD::SRL: - case ISD::SRA: return LowerSRx(Op, DAG, Subtarget); - case ISD::FORMAL_ARGUMENTS: - return LowerFORMAL_ARGUMENTS(Op, DAG); + case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG); case ISD::RETURNADDR: break; case ISD::FRAMEADDR: break; case ISD::GLOBAL_OFFSET_TABLE: return LowerGLOBAL_OFFSET_TABLE(Op, DAG); case ISD::MEMCPY: return LowerMEMCPY(Op, DAG); case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); + + + // FIXME: Remove these when LegalizeDAGTypes lands. + case ISD::BIT_CONVERT: return SDOperand(ExpandBIT_CONVERT(Op.Val, DAG), 0); + case ISD::SRL: + case ISD::SRA: return SDOperand(ExpandSRx(Op.Val, DAG,Subtarget),0); } return SDOperand(); } + +/// ExpandOperationResult - Provide custom lowering hooks for expanding +/// operations. +SDNode *ARMTargetLowering::ExpandOperationResult(SDNode *N, SelectionDAG &DAG) { + switch (N->getOpcode()) { + default: assert(0 && "Don't know how to custom expand this!"); abort(); + case ISD::BIT_CONVERT: return ExpandBIT_CONVERT(N, DAG); + case ISD::SRL: + case ISD::SRA: return ExpandSRx(N, DAG, Subtarget); + } +} + + //===----------------------------------------------------------------------===// // ARM Scheduler Hooks //===----------------------------------------------------------------------===// diff --git a/lib/Target/ARM/ARMISelLowering.h b/lib/Target/ARM/ARMISelLowering.h index e6fb9458b6c..93971c514b8 100644 --- a/lib/Target/ARM/ARMISelLowering.h +++ b/lib/Target/ARM/ARMISelLowering.h @@ -76,6 +76,8 @@ namespace llvm { explicit ARMTargetLowering(TargetMachine &TM); virtual SDOperand LowerOperation(SDOperand Op, SelectionDAG &DAG); + virtual SDNode *ExpandOperationResult(SDNode *N, SelectionDAG &DAG); + virtual const char *getTargetNodeName(unsigned Opcode) const; virtual MachineBasicBlock *InsertAtEndOfBasicBlock(MachineInstr *MI, diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index 9a86561f0dc..0482dad4aae 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -1258,7 +1258,7 @@ X86TargetLowering::LowerMemOpCallTo(SDOperand Op, SelectionDAG &DAG, SDOperand AlignNode = DAG.getConstant(Align, MVT::i32); SDOperand SizeNode = DAG.getConstant(Size, MVT::i32); - SDOperand AlwaysInline = DAG.getConstant(1, MVT::i1); + SDOperand AlwaysInline = DAG.getConstant(1, MVT::i32); return DAG.getMemcpy(Chain, PtrOff, Arg, SizeNode, AlignNode, AlwaysInline); @@ -3918,22 +3918,22 @@ SDOperand X86TargetLowering::LowerSINT_TO_FP(SDOperand Op, SelectionDAG &DAG) { return Result; } -SDOperand X86TargetLowering::LowerFP_TO_SINT(SDOperand Op, SelectionDAG &DAG) { +std::pair X86TargetLowering:: +FP_TO_SINTHelper(SDOperand Op, SelectionDAG &DAG) { assert(Op.getValueType() <= MVT::i64 && Op.getValueType() >= MVT::i16 && "Unknown FP_TO_SINT to lower!"); - SDOperand Result; // These are really Legal. if (Op.getValueType() == MVT::i32 && X86ScalarSSEf32 && Op.getOperand(0).getValueType() == MVT::f32) - return Result; + return std::make_pair(SDOperand(), SDOperand()); if (Op.getValueType() == MVT::i32 && X86ScalarSSEf64 && Op.getOperand(0).getValueType() == MVT::f64) - return Result; + return std::make_pair(SDOperand(), SDOperand()); if (Subtarget->is64Bit() && Op.getValueType() == MVT::i64 && Op.getOperand(0).getValueType() != MVT::f80) - return Result; + return std::make_pair(SDOperand(), SDOperand()); // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary // stack slot. @@ -3943,10 +3943,10 @@ SDOperand X86TargetLowering::LowerFP_TO_SINT(SDOperand Op, SelectionDAG &DAG) { SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); unsigned Opc; switch (Op.getValueType()) { - default: assert(0 && "Invalid FP_TO_SINT to lower!"); - case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; - case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; - case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; + default: assert(0 && "Invalid FP_TO_SINT to lower!"); + case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; + case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; + case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; } SDOperand Chain = DAG.getEntryNode(); @@ -3969,20 +3969,33 @@ SDOperand X86TargetLowering::LowerFP_TO_SINT(SDOperand Op, SelectionDAG &DAG) { SDOperand Ops[] = { Chain, Value, StackSlot }; SDOperand FIST = DAG.getNode(Opc, MVT::Other, Ops, 3); - // Load the result. If this is an i64 load on an x86-32 host, expand the - // load. - if (Op.getValueType() != MVT::i64 || Subtarget->is64Bit()) - return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0); - - SDOperand Lo = DAG.getLoad(MVT::i32, FIST, StackSlot, NULL, 0); - StackSlot = DAG.getNode(ISD::ADD, StackSlot.getValueType(), StackSlot, - DAG.getConstant(StackSlot.getValueType(), 4)); - SDOperand Hi = DAG.getLoad(MVT::i32, FIST, StackSlot, NULL, 0); - - - return DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi); + return std::make_pair(FIST, StackSlot); } +SDOperand X86TargetLowering::LowerFP_TO_SINT(SDOperand Op, SelectionDAG &DAG) { + assert((Op.getValueType() != MVT::i64 || Subtarget->is64Bit()) && + "This FP_TO_SINT must be expanded!"); + + std::pair Vals = FP_TO_SINTHelper(Op, DAG); + SDOperand FIST = Vals.first, StackSlot = Vals.second; + if (FIST.Val == 0) return SDOperand(); + + // Load the result. + return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0); +} + +SDNode *X86TargetLowering::ExpandFP_TO_SINT(SDNode *N, SelectionDAG &DAG) { + std::pair Vals = FP_TO_SINTHelper(SDOperand(N, 0), DAG); + SDOperand FIST = Vals.first, StackSlot = Vals.second; + if (FIST.Val == 0) return 0; + + // Return an i64 load from the stack slot. + SDOperand Res = DAG.getLoad(MVT::i64, FIST, StackSlot, NULL, 0); + + // Use a MERGE_VALUES node to drop the chain result value. + return DAG.getNode(ISD::MERGE_VALUES, MVT::i64, Res).Val; +} + SDOperand X86TargetLowering::LowerFABS(SDOperand Op, SelectionDAG &DAG) { MVT::ValueType VT = Op.getValueType(); MVT::ValueType EltVT = VT; @@ -4587,32 +4600,36 @@ SDOperand X86TargetLowering::LowerMEMCPYInline(SDOperand Chain, return Chain; } -SDOperand -X86TargetLowering::LowerREADCYCLCECOUNTER(SDOperand Op, SelectionDAG &DAG) { +/// Expand the result of: i64,outchain = READCYCLECOUNTER inchain +SDNode *X86TargetLowering::ExpandREADCYCLECOUNTER(SDNode *N, SelectionDAG &DAG){ SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); - SDOperand TheOp = Op.getOperand(0); - SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheOp, 1); + SDOperand TheChain = N->getOperand(0); + SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheChain, 1); if (Subtarget->is64Bit()) { - SDOperand Copy1 = - DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1)); - SDOperand Copy2 = DAG.getCopyFromReg(Copy1.getValue(1), X86::RDX, - MVT::i64, Copy1.getValue(2)); - SDOperand Tmp = DAG.getNode(ISD::SHL, MVT::i64, Copy2, + SDOperand rax = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1)); + SDOperand rdx = DAG.getCopyFromReg(rax.getValue(1), X86::RDX, + MVT::i64, rax.getValue(2)); + SDOperand Tmp = DAG.getNode(ISD::SHL, MVT::i64, rdx, DAG.getConstant(32, MVT::i8)); SDOperand Ops[] = { - DAG.getNode(ISD::OR, MVT::i64, Copy1, Tmp), Copy2.getValue(1) + DAG.getNode(ISD::OR, MVT::i64, rax, Tmp), rdx.getValue(1) }; Tys = DAG.getVTList(MVT::i64, MVT::Other); - return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops, 2); + return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops, 2).Val; } - SDOperand Copy1 = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)); - SDOperand Copy2 = DAG.getCopyFromReg(Copy1.getValue(1), X86::EDX, - MVT::i32, Copy1.getValue(2)); - SDOperand Ops[] = { Copy1, Copy2, Copy2.getValue(1) }; - Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); - return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops, 3); + SDOperand eax = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)); + SDOperand edx = DAG.getCopyFromReg(eax.getValue(1), X86::EDX, + MVT::i32, eax.getValue(2)); + // Use a buildpair to merge the two 32-bit values into a 64-bit one. + SDOperand Ops[] = { eax, edx }; + Ops[0] = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Ops, 2); + + // Use a MERGE_VALUES to return the value and chain. + Ops[1] = edx.getValue(1); + Tys = DAG.getVTList(MVT::i64, MVT::Other); + return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops, 2).Val; } SDOperand X86TargetLowering::LowerVASTART(SDOperand Op, SelectionDAG &DAG) { @@ -5032,7 +5049,6 @@ SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG); case ISD::MEMSET: return LowerMEMSET(Op, DAG); case ISD::MEMCPY: return LowerMEMCPY(Op, DAG); - case ISD::READCYCLECOUNTER: return LowerREADCYCLCECOUNTER(Op, DAG); case ISD::VASTART: return LowerVASTART(Op, DAG); case ISD::VACOPY: return LowerVACOPY(Op, DAG); case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); @@ -5044,8 +5060,21 @@ SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); case ISD::TRAMPOLINE: return LowerTRAMPOLINE(Op, DAG); case ISD::FLT_ROUNDS: return LowerFLT_ROUNDS(Op, DAG); + + + // FIXME: REMOVE THIS WHEN LegalizeDAGTypes lands. + case ISD::READCYCLECOUNTER: + return SDOperand(ExpandREADCYCLECOUNTER(Op.Val, DAG), 0); + } +} + +/// ExpandOperation - Provide custom lowering hooks for expanding operations. +SDNode *X86TargetLowering::ExpandOperationResult(SDNode *N, SelectionDAG &DAG) { + switch (N->getOpcode()) { + default: assert(0 && "Should not custom lower this!"); + case ISD::FP_TO_SINT: return ExpandFP_TO_SINT(N, DAG); + case ISD::READCYCLECOUNTER: return ExpandREADCYCLECOUNTER(N, DAG); } - return SDOperand(); } const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h index ac16ff42478..5fe49f8d81b 100644 --- a/lib/Target/X86/X86ISelLowering.h +++ b/lib/Target/X86/X86ISelLowering.h @@ -321,6 +321,12 @@ namespace llvm { /// virtual SDOperand LowerOperation(SDOperand Op, SelectionDAG &DAG); + /// ExpandOperation - Custom lower the specified operation, splitting the + /// value into two pieces. + /// + virtual SDNode *ExpandOperationResult(SDNode *N, SelectionDAG &DAG); + + virtual SDOperand PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const; virtual MachineBasicBlock *InsertAtEndOfBasicBlock(MachineInstr *MI, @@ -444,6 +450,9 @@ namespace llvm { SDOperand LowerFastCCArguments(SDOperand Op, SelectionDAG &DAG); SDOperand LowerFastCCCallTo(SDOperand Op, SelectionDAG &DAG, unsigned CC); + std::pair FP_TO_SINTHelper(SDOperand Op, + SelectionDAG &DAG); + SDOperand LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG); SDOperand LowerVECTOR_SHUFFLE(SDOperand Op, SelectionDAG &DAG); SDOperand LowerEXTRACT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG); @@ -471,7 +480,6 @@ namespace llvm { SDOperand LowerRET(SDOperand Op, SelectionDAG &DAG); SDOperand LowerDYNAMIC_STACKALLOC(SDOperand Op, SelectionDAG &DAG); SDOperand LowerFORMAL_ARGUMENTS(SDOperand Op, SelectionDAG &DAG); - SDOperand LowerREADCYCLCECOUNTER(SDOperand Op, SelectionDAG &DAG); SDOperand LowerVASTART(SDOperand Op, SelectionDAG &DAG); SDOperand LowerVACOPY(SDOperand Op, SelectionDAG &DAG); SDOperand LowerINTRINSIC_WO_CHAIN(SDOperand Op, SelectionDAG &DAG); @@ -481,6 +489,8 @@ namespace llvm { SDOperand LowerEH_RETURN(SDOperand Op, SelectionDAG &DAG); SDOperand LowerTRAMPOLINE(SDOperand Op, SelectionDAG &DAG); SDOperand LowerFLT_ROUNDS(SDOperand Op, SelectionDAG &DAG); + SDNode *ExpandFP_TO_SINT(SDNode *N, SelectionDAG &DAG); + SDNode *ExpandREADCYCLECOUNTER(SDNode *N, SelectionDAG &DAG); }; }