@@ -7311,8 +7311,6 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
73117311 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
73127312 // needing to pad the source vector, because the inactive lanes can simply be
73137313 // disabled and not contribute to the result.
7314- // TODO: VECREDUCE_FADD, VECREDUCE_FMUL aren't currently mapped correctly,
7315- // and thus don't take this path.
73167314 if (auto VPOpcode = ISD::getVPForBaseOpcode (Opc);
73177315 VPOpcode && TLI.isOperationLegalOrCustom (*VPOpcode, WideVT)) {
73187316 SDValue Start = NeutralElem;
@@ -7351,6 +7349,7 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
73517349 SDValue VecOp = N->getOperand (1 );
73527350 SDValue Op = GetWidenedVector (VecOp);
73537351
7352+ EVT VT = N->getValueType (0 );
73547353 EVT OrigVT = VecOp.getValueType ();
73557354 EVT WideVT = Op.getValueType ();
73567355 EVT ElemVT = OrigVT.getVectorElementType ();
@@ -7364,6 +7363,19 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
73647363 unsigned OrigElts = OrigVT.getVectorMinNumElements ();
73657364 unsigned WideElts = WideVT.getVectorMinNumElements ();
73667365
7366+ // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
7367+ // needing to pad the source vector, because the inactive lanes can simply be
7368+ // disabled and not contribute to the result.
7369+ if (auto VPOpcode = ISD::getVPForBaseOpcode (Opc);
7370+ VPOpcode && TLI.isOperationLegalOrCustom (*VPOpcode, WideVT)) {
7371+ EVT WideMaskVT = EVT::getVectorVT (*DAG.getContext (), MVT::i1,
7372+ WideVT.getVectorElementCount ());
7373+ SDValue Mask = DAG.getAllOnesConstant (dl, WideMaskVT);
7374+ SDValue EVL = DAG.getElementCount (dl, TLI.getVPExplicitVectorLengthTy (),
7375+ OrigVT.getVectorElementCount ());
7376+ return DAG.getNode (*VPOpcode, dl, VT, {AccOp, Op, Mask, EVL}, Flags);
7377+ }
7378+
73677379 if (WideVT.isScalableVector ()) {
73687380 unsigned GCD = std::gcd (OrigElts, WideElts);
73697381 EVT SplatVT = EVT::getVectorVT (*DAG.getContext (), ElemVT,
@@ -7372,14 +7384,14 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
73727384 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
73737385 Op = DAG.getNode (ISD::INSERT_SUBVECTOR, dl, WideVT, Op, SplatNeutral,
73747386 DAG.getVectorIdxConstant (Idx, dl));
7375- return DAG.getNode (Opc, dl, N-> getValueType ( 0 ) , AccOp, Op, Flags);
7387+ return DAG.getNode (Opc, dl, VT , AccOp, Op, Flags);
73767388 }
73777389
73787390 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
73797391 Op = DAG.getNode (ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
73807392 DAG.getVectorIdxConstant (Idx, dl));
73817393
7382- return DAG.getNode (Opc, dl, N-> getValueType ( 0 ) , AccOp, Op, Flags);
7394+ return DAG.getNode (Opc, dl, VT , AccOp, Op, Flags);
73837395}
73847396
73857397SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE (SDNode *N) {
0 commit comments