Arm64: Combine if conditions into compare chains

src/coreclr/jit/codegen.h

@@ -887,6 +887,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
     void genCodeForContainedCompareChain(GenTree* tree, bool* inchain, GenCondition* prevCond);
 #endif
     void genCodeForSelect(GenTreeOp* select);
+    void genCodeForAndFlags(GenTreeOp* select);
     void genIntrinsic(GenTreeIntrinsic* treeNode);
     void genPutArgStk(GenTreePutArgStk* treeNode);
     void genPutArgReg(GenTreeOp* tree);

src/coreclr/jit/codegenarm64.cpp

@@ -4784,6 +4784,48 @@ void CodeGen::genCodeForSelect(GenTreeOp* tree)
     genProduceReg(tree);
 }
 
+//------------------------------------------------------------------------
+// genCodeForAndFlags: Generates code for ANDFLAGS statement.
+//
+// Arguments:
+//    tree - the node
+//
+void CodeGen::genCodeForAndFlags(GenTreeOp* tree)
+{
+    var_types targetType = tree->TypeGet();
+    emitter*  emit       = GetEmitter();
+
+    assert(tree->OperIs(GT_ANDFLAGS));
+    assert(tree->GetRegNum() == REG_NA);
+
+    GenTree* op1 = tree->gtGetOp1();
+    GenTree* op2 = tree->gtGetOp2();
+
+    assert (tree->isContainedCompareChainSegment(op2));
+
+    GenCondition cond;
+    bool         chain = false;
+
+    JITDUMP("Generating compare chain:\n");
+    if (op1->isContained())
+    {
+        // Generate Op1 into flags.
+        genCodeForContainedCompareChain(op1, &chain, &cond);
+        assert(chain);
+    }
+    else
+    {
+        // Op1 is not contained, move it from a register into flags.
+        emit->emitIns_R_I(INS_cmp, emitActualTypeSize(op1), op1->GetRegNum(), 0);
+        cond  = GenCondition::NE;
+        chain = true;
+    }
+
+    // Gen Op2 into flags.
+    genCodeForContainedCompareChain(op2, &chain, &cond);
+    assert(chain);
+}
+
 //------------------------------------------------------------------------
 // genCodeForJumpCompare: Generates code for jmpCompare statement.
 //

src/coreclr/jit/codegenarmarch.cpp

@@ -366,6 +366,11 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
         case GT_SELECT:
             genCodeForSelect(treeNode->AsConditional());
             break;
+
+        case GT_ANDFLAGS:
+            genConsumeOperands(treeNode->AsOp());
+            genCodeForAndFlags(treeNode->AsOp());
+            break;
 #endif
 
         case GT_JTRUE:

src/coreclr/jit/codegenlinear.cpp

@@ -2592,8 +2592,43 @@ void CodeGen::genCodeForJumpTrue(GenTreeOp* jtrue)
     assert(compiler->compCurBB->bbJumpKind == BBJ_COND);
     assert(jtrue->OperIs(GT_JTRUE));
 
-    GenTreeOp*   relop     = jtrue->gtGetOp1()->AsOp();
-    GenCondition condition = GenCondition::FromRelop(relop);
+    GenTreeOp*   relop = jtrue->gtGetOp1()->AsOp();
+    GenCondition condition;
+
+    // Operands should never be contained inside a jtrue.
+    assert(!relop->isContained());
+
+#if defined(TARGET_ARM64)
+    if (relop->OperIs(GT_AND))
+    {
+        // The condition was generated into a register.
+        assert(relop->gtType != TYP_VOID);
+        regNumber reg = relop->GetRegNum();
+        assert(reg != REG_NA);
+        emitAttr attr = emitActualTypeSize(relop->TypeGet());
+        GetEmitter()->emitIns_J_R(INS_cbnz, attr, compiler->compCurBB->bbJumpDest, reg);
+        return;
+    }
+    else if (relop->OperIs(GT_ANDFLAGS))
+    {
+        // Find the last contained compare in the chain.
+        assert(relop->gtType == TYP_VOID);
+        GenTreeOp* lastCompare = relop->gtGetOp2()->AsOp();
+        assert(lastCompare->isContained());
+        while (!lastCompare->OperIsCompare())
+        {
+            assert(lastCompare->OperIs(GT_AND));
+            lastCompare = lastCompare->gtGetOp2()->AsOp();
+            assert(lastCompare->isContained());
+        }
+        condition = GenCondition::FromRelop(lastCompare);
+    }
+    else
+#endif
+    {
+        assert(relop->OperIsCompare());
+        condition = GenCondition::FromRelop(relop);
+    }
 
     if (condition.PreferSwap())
     {

src/coreclr/jit/compiler.h

@@ -2870,6 +2870,12 @@ class Compiler
     // before they have been set.)
     bool gtComplexityExceeds(GenTree* tree, unsigned limit);
 
+    // Can the test condition be reversed without creating a new node.
+    bool gtCanReverseCondSimple(GenTree* tree)
+    {
+        return (tree->OperIsCompare() || tree->OperIs(GT_JCC, GT_SETCC) || tree->OperIs(GT_JCMP));
+    }
+
     GenTree* gtReverseCond(GenTree* tree);
 
     static bool gtHasRef(GenTree* tree, unsigned lclNum);

src/coreclr/jit/fgopt.cpp

@@ -5172,6 +5172,18 @@ bool Compiler::fgReorderBlocks(bool useProfile)
             }
         }
 
+        // Cannot handle cases where reversing creates a new node.
+        if (bPrev->bbJumpKind == BBJ_COND)
+        {
+            Statement* const condTestStmt = bPrev->lastStmt();
+            GenTree* const   condTest     = condTestStmt->GetRootNode();
+            noway_assert(condTest->gtOper == GT_JTRUE);
+            if (!gtCanReverseCondSimple(condTest->gtGetOp1()))
+            {
+                reorderBlock = false;
+            }
+        }
+
         if (reorderBlock == false)
         {
             //
@@ -6163,6 +6175,13 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication, bool isPhase)
                         }
                     }
 
+                    // Cannot handle cases where reversing creates a new node.
+                    GenTree* last = block->lastNode();
+                    if (last->OperGet() == GT_JTRUE && !gtCanReverseCondSimple(last->gtGetOp1()))
+                    {
+                        optimizeJump = false;
+                    }
+
                     if (optimizeJump && isJumpToJoinFree)
                     {
                         // In the join free case, we also need to move bDest right after bNext

src/coreclr/jit/gentree.cpp

@@ -17132,8 +17132,7 @@ bool GenTree::canBeContained() const
         return false;
     }
 
-    // It is not possible for nodes that do not produce values or that are not containable values to be contained.
-    if (!IsValue() || ((DebugOperKind() & DBK_NOCONTAIN) != 0) || (OperIsHWIntrinsic() && !isContainableHWIntrinsic()))
+    if (((DebugOperKind() & DBK_NOCONTAIN) != 0) || (OperIsHWIntrinsic() && !isContainableHWIntrinsic()))
     {
         return false;
     }

src/coreclr/jit/gentree.h

@@ -910,11 +910,14 @@ struct GenTree
         return isContained() && IsCnsIntOrI() && !isUsedFromSpillTemp();
     }
 
+#if defined(TARGET_ARM64)
     // Node and its child in isolation form a contained compare chain.
     bool isContainedCompareChainSegment(GenTree* child) const
     {
-        return (OperIs(GT_AND) && child->isContained() && (child->OperIs(GT_AND) || child->OperIsCmpCompare()));
+        return ((OperIs(GT_AND) || OperIs(GT_ANDFLAGS)) && child->isContained() &&
+                (child->OperIs(GT_AND) || child->OperIsCmpCompare()));
     }
+#endif
 
     bool isContainedFltOrDblImmed() const
     {
@@ -1089,8 +1092,8 @@ struct GenTree
         if (gtType == TYP_VOID)
         {
             // These are the only operators which can produce either VOID or non-VOID results.
-            assert(OperIs(GT_NOP, GT_CALL, GT_COMMA) || OperIsCompare() || OperIsLong() || OperIsHWIntrinsic() ||
-                   IsCnsVec());
+            assert(OperIs(GT_NOP, GT_CALL, GT_COMMA, GT_AND) || OperIsCompare() || OperIsLong() ||
+                   OperIsHWIntrinsic() || IsCnsVec());
             return false;
         }
 

src/coreclr/jit/gtlist.h

@@ -242,6 +242,10 @@ GTNODE(SETCC            , GenTreeCC          ,0,GTK_LEAF|DBK_NOTHIR)
 // The XARCH BT instruction. Like CMP, this sets the condition flags (CF to be precise) and does not produce a value.
 GTNODE(BT               , GenTreeOp          ,0,(GTK_BINOP|GTK_NOVALUE|DBK_NOTHIR))
 #endif
+// Sets the condition flags according to the combined results of its children.
+#if defined(TARGET_ARM64)
+GTNODE(ANDFLAGS         , GenTreeOp          ,0,GTK_BINOP|GTK_NOVALUE|DBK_NOTHIR)
+#endif
 
 //-----------------------------------------------------------------------------
 //  Other nodes that look like unary/binary operators:

src/coreclr/jit/ifconversion.cpp

@@ -558,10 +558,11 @@ bool OptIfConversionDsc::optIfConvert()
     }
 
     // Verify the test block ends with a condition that we can manipulate.
+    // Assuming that an AND connected to a JTRUE is always a valid chain.
     GenTree* last = m_startBlock->lastStmt()->GetRootNode();
     noway_assert(last->OperIs(GT_JTRUE));
     m_cond = last->gtGetOp1();
-    if (!m_cond->OperIsCompare())
+    if (!m_cond->OperIsCompare() && !m_cond->OperIs(GT_AND))
     {
         return false;
     }
@@ -668,22 +669,16 @@ bool OptIfConversionDsc::optIfConvert()
     {
         if (m_doElseConversion)
         {
-            selectTrueInput  = m_elseOperation.node->gtGetOp2();
-            selectFalseInput = m_thenOperation.node->gtGetOp2();
+            selectTrueInput = m_elseOperation.node->gtGetOp2();
         }
         else
         {
-            // Invert the condition (to help matching condition codes back to CIL).
-            GenTree* revCond = m_comp->gtReverseCond(m_cond);
-            assert(m_cond == revCond); // Ensure `gtReverseCond` did not create a new node.
-
             // Duplicate the destination of the Then assignment.
             assert(m_thenOperation.node->gtGetOp1()->IsLocal());
-            selectFalseInput = m_comp->gtCloneExpr(m_thenOperation.node->gtGetOp1());
-            selectFalseInput->gtFlags &= GTF_EMPTY;
-
-            selectTrueInput = m_thenOperation.node->gtGetOp2();
+            selectTrueInput = m_comp->gtCloneExpr(m_thenOperation.node->gtGetOp1());
+            selectTrueInput->gtFlags &= GTF_EMPTY;
         }
+        selectFalseInput = m_thenOperation.node->gtGetOp2();
 
         // Pick the type as the type of the local, which should always be compatible even for implicit coercions.
         selectType = genActualType(m_thenOperation.node->gtGetOp1());

src/coreclr/jit/lower.cpp

@@ -7164,10 +7164,31 @@ void Lowering::ContainCheckRet(GenTreeUnOp* ret)
 //
 void Lowering::ContainCheckJTrue(GenTreeOp* node)
 {
-    // The compare does not need to be generated into a register.
-    GenTree* cmp = node->gtGetOp1();
-    cmp->gtType  = TYP_VOID;
-    cmp->gtFlags |= GTF_SET_FLAGS;
+    GenTree* op1 = node->gtGetOp1();
+
+    if (op1->OperIsCompare())
+    {
+        // The compare does not need to be generated into a register.
+        op1->gtType = TYP_VOID;
+        op1->gtFlags |= GTF_SET_FLAGS;
+    }
+#if defined(TARGET_ARM64)
+    else if (op1->OperIs(GT_AND))
+    {
+        // If the second op of the AND is contained, then the AND does not need to be generated
+        // into a register.
+        if (op1->gtGetOp2()->isContained())
+        {
+            op1->SetOper(GT_ANDFLAGS);
+            op1->gtType = TYP_VOID;
+            op1->gtFlags |= GTF_SET_FLAGS;
+        }
+    }
+#endif
+    else
+    {
+        unreached();
+    }
 }
 
 //------------------------------------------------------------------------

src/coreclr/jit/lowerarmarch.cpp

@@ -2281,7 +2281,8 @@ bool Lowering::IsValidCompareChain(GenTree* child, GenTree* parent)
         return IsValidCompareChain(child->AsOp()->gtGetOp2(), child) &&
                IsValidCompareChain(child->AsOp()->gtGetOp1(), child);
     }
-    else if (child->OperIsCmpCompare() && varTypeIsIntegral(child->gtGetOp1()) && varTypeIsIntegral(child->gtGetOp2()))
+    else if (child->OperIsCmpCompare() && varTypeIsIntegralOrI(child->gtGetOp1()) &&
+             varTypeIsIntegralOrI(child->gtGetOp2()))
     {
         // Can the child compare be contained.
         return IsSafeToContainMem(parent, child);
@@ -2307,7 +2308,6 @@ bool Lowering::IsValidCompareChain(GenTree* child, GenTree* parent)
 bool Lowering::ContainCheckCompareChain(GenTree* child, GenTree* parent, GenTree** startOfChain)
 {
     assert(parent->OperIs(GT_AND) || parent->OperIs(GT_SELECT));
-    *startOfChain = nullptr; // Nothing found yet.
 
     if (parent->isContainedCompareChainSegment(child))
     {
@@ -2320,23 +2320,49 @@ bool Lowering::ContainCheckCompareChain(GenTree* child, GenTree* parent, GenTree
         if (child->OperIs(GT_AND))
         {
             // If Op2 is not contained, then try to contain it.
-            if (!child->isContainedCompareChainSegment(child->AsOp()->gtGetOp2()))
+            if (!child->isContainedCompareChainSegment(child->gtGetOp2()))
             {
                 if (!ContainCheckCompareChain(child->gtGetOp2(), child, startOfChain))
                 {
                     // Op2 must be contained in order to contain Op1 or the AND.
                     return false;
                 }
             }
+            else
+            {
+                // Start of the chain is unknown.
+                *startOfChain = nullptr;
+
+                if (child->gtGetOp2()->OperIsCmpCompare())
+                {
+                    // Ensure the children of the compare are contained correctly.
+                    child->gtGetOp2()->gtGetOp1()->ClearContained();
+                    child->gtGetOp2()->gtGetOp2()->ClearContained();
+                    ContainCheckConditionalCompare(child->gtGetOp2()->AsOp());
+                }
+            }
 
             // If Op1 is not contained, then try to contain it.
-            if (!child->isContainedCompareChainSegment(child->AsOp()->gtGetOp1()))
+            if (!child->isContainedCompareChainSegment(child->gtGetOp1()))
             {
                 if (!ContainCheckCompareChain(child->gtGetOp1(), child, startOfChain))
                 {
                     return false;
                 }
             }
+            else
+            {
+                // Start of the chain is unknown.
+                *startOfChain = nullptr;
+
+                if (child->gtGetOp1()->OperIsCmpCompare())
+                {
+                    // Ensure the children of the compare are contained correctly.
+                    child->gtGetOp1()->gtGetOp1()->ClearContained();
+                    child->gtGetOp1()->gtGetOp2()->ClearContained();
+                    ContainCheckConditionalCompare(child->gtGetOp1()->AsOp());
+                }
+            }
 
             // Contain the AND.
             child->SetContained();
@@ -2382,17 +2408,15 @@ void Lowering::ContainCheckCompareChainForAnd(GenTree* tree)
         // only be contained if Op2 is contained.
         if (ContainCheckCompareChain(tree->AsOp()->gtGetOp2(), tree, &startOfChain))
         {
-            if (ContainCheckCompareChain(tree->AsOp()->gtGetOp1(), tree, &startOfChain))
+            ContainCheckCompareChain(tree->AsOp()->gtGetOp1(), tree, &startOfChain);
+
+            // The earliest node in the chain will be generated as a standard compare.
+            if (startOfChain != nullptr)
             {
-                // If op1 is the start of a chain, then it'll be generated as a standard compare.
-                if (startOfChain != nullptr)
-                {
-                    // The earliest node in the chain will be generated as a standard compare.
-                    assert(startOfChain->OperIsCmpCompare());
-                    startOfChain->AsOp()->gtGetOp1()->ClearContained();
-                    startOfChain->AsOp()->gtGetOp2()->ClearContained();
-                    ContainCheckCompare(startOfChain->AsOp());
-                }
+                assert(startOfChain->OperIsCmpCompare());
+                startOfChain->AsOp()->gtGetOp1()->ClearContained();
+                startOfChain->AsOp()->gtGetOp2()->ClearContained();
+                ContainCheckCompare(startOfChain->AsOp());
             }
         }