Ensure that we add the broadcast and mask flags prior to computing the instruction size (#105757)

Author: tannergooding

src/coreclr/jit/emitxarch.cpp

@@ -7464,6 +7464,9 @@ void emitter::emitIns_R_C(
         else
 #endif // TARGET_X86
         {
+            SetEvexBroadcastIfNeeded(id, instOptions);
+            SetEvexEmbMaskIfNeeded(id, instOptions);
+
             sz = emitInsSizeCV(id, insCodeRM(ins));
         }
 
@@ -7478,9 +7481,6 @@ void emitter::emitIns_R_C(
         }
     }
 
-    SetEvexBroadcastIfNeeded(id, instOptions);
-    SetEvexEmbMaskIfNeeded(id, instOptions);
-
     id->idCodeSize(sz);
 
     id->idAddr()->iiaFieldHnd = fldHnd;
@@ -9295,8 +9295,7 @@ void emitter::emitIns_R_S(instruction ins, emitAttr attr, regNumber ireg, int va
         return;
     }
 
-    instrDesc*     id = emitNewInstr(attr);
-    UNATIVE_OFFSET sz;
+    instrDesc* id = emitNewInstr(attr);
     id->idIns(ins);
     id->idInsFmt(fmt);
     id->idReg1(ireg);
@@ -9305,7 +9304,7 @@ void emitter::emitIns_R_S(instruction ins, emitAttr attr, regNumber ireg, int va
     SetEvexBroadcastIfNeeded(id, instOptions);
     SetEvexEmbMaskIfNeeded(id, instOptions);
 
-    sz = emitInsSizeSV(id, insCodeRM(ins), varx, offs);
+    UNATIVE_OFFSET sz = emitInsSizeSV(id, insCodeRM(ins), varx, offs);
     id->idCodeSize(sz);
 #ifdef DEBUG
     id->idDebugOnlyInfo()->idVarRefOffs = emitVarRefOffs;

src/tests/JIT/Regression/JitBlue/Runtime_105693/Runtime_105693.cs

@@ -0,0 +1,46 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Runtime.CompilerServices;
+using System.Numerics;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
+using Xunit;
+
+// Generated by Fuzzlyn v2.2 on 2024-07-29 22:32:33
+// Run on X64 Linux
+// Seed: 1257362736509498015-vectort,vector128,vector256,x86aes,x86avx,x86avx2,x86avx512bw,x86avx512bwvl,x86avx512cd,x86avx512cdvl,x86avx512dq,x86avx512dqvl,x86avx512f,x86avx512fvl,x86avx512fx64,x86bmi1,x86bmi1x64,x86bmi2,x86bmi2x64,x86fma,x86lzcnt,x86lzcntx64,x86pclmulqdq,x86popcnt,x86popcntx64,x86sse,x86ssex64,x86sse2,x86sse2x64,x86sse3,x86sse41,x86sse41x64,x86sse42,x86sse42x64,x86ssse3,x86x86base
+// Reduced from 29.1 KiB to 0.6 KiB in 00:00:26
+// Hits JIT assert in Release:
+// Assertion failed 'estimatedSize >= actualSize' in 'Program:Main(Fuzzlyn.ExecutionServer.IRuntime)' during 'Emit code' (IL size 59; hash 0xade6b36b; FullOpts)
+//
+//     File: /__w/1/s/src/coreclr/jit/emit.cpp Line: 4361
+//
+
+public class Runtime_105693
+{
+    public static int s_2;
+
+    [Fact]
+    public static void TestEntryPoint()
+    {
+        if (Avx2.IsSupported)
+        {
+            var vr5 = Vector128.Create<int>(1);
+            var vr6 = Avx2.BroadcastScalarToVector256(vr5);
+            var vr7 = Avx2.Abs(vr6);
+            var vr8 = Vector256.CreateScalar(1U);
+            if (Avx.TestZ(vr7, vr8))
+            {
+                var vr9 = Vector128.CreateScalar(1f);
+                s_2 = Sse.ConvertToInt32(vr9);
+            }
+            else
+            {
+                s_2 = 1;
+            }
+            Assert.Equal(1, s_2);
+        }
+    }
+}

src/tests/JIT/Regression/JitBlue/Runtime_105693/Runtime_105693.csproj

@@ -0,0 +1,8 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <Optimize>True</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildProjectName).cs" />
+  </ItemGroup>
+</Project>