[ci] Automatically retry failed native unit test failures once (#1095)

jpobst · web-flow · commit 554d8193f2d3 · 2023-04-11T08:18:15.000-04:00
Context: #514 There is a known native crash that occasionally occurs when running unit tests on CI; see Issue #514: * Assertion: should not be reached at class-accessors.c:81 ================================================================= Native Crash Reporting ================================================================= Got a SIGABRT while executing native code. This usually indicates a fatal error in the mono runtime or one of the native libraries used by your application. ================================================================= ================================================================= Native stacktrace: ================================================================= 0x102d2091b - /Library/Frameworks/Mono.framework/Versions/Current/Commands/mono : mono_dump_native_crash_info 0x102cb97b5 - /Library/Frameworks/Mono.framework/Versions/Current/Commands/mono : mono_handle_native_crash 0x102d1fe71 - /Library/Frameworks/Mono.framework/Versions/Current/Commands/mono : sigabrt_signal_handler 0x7fff79156b5d - /usr/lib/system/libsystem_platform.dylib : _sigtramp 0x10a126f87 - Unknown 0x7fff790106a6 - /usr/lib/system/libsystem_c.dylib : abort 0x102f28837 - /Library/Frameworks/Mono.framework/Versions/Current/Commands/mono : monoeg_assert_abort 0x102f09a9f - /Library/Frameworks/Mono.framework/Versions/Current/Commands/mono : mono_log_write_logfile 0x102f28bbe - /Library/Frameworks/Mono.framework/Versions/Current/Commands/mono : monoeg_g_logv_nofree 0x102f28d3f - /Library/Frameworks/Mono.framework/Versions/Current/Commands/mono : monoeg_assertion_message 0x102db6ee5 - /Library/Frameworks/Mono.framework/Versions/Current/Commands/mono : mono_class_get_flags 0x102dac83b - /Library/Frameworks/Mono.framework/Versions/Current/Commands/mono : mono_class_setup_fields 0x102dacdc1 - /Library/Frameworks/Mono.framework/Versions/Current/Commands/mono : mono_class_init_internal 0x102da7e7f - /Library/Frameworks/Mono.framework/Versions/Current/Commands/mono : mono_class_is_subclass_of_internal 0x102e2e4f3 - /Library/Frameworks/Mono.framework/Versions/Current/Commands/mono : mono_class_is_subclass_of 0x106bf1348 - /Users/runner/runners/2.160.0/work/1/s/bin/TestDebug/libjava-interop.dylib : _ZL19get_gc_bridge_indexP19JavaInteropGCBridgeP10_MonoClass 0x106bf0a4c - /Users/runner/runners/2.160.0/work/1/s/bin/TestDebug/libjava-interop.dylib : _ZL20gc_bridge_class_kindP10_MonoClass SIGQUIT: [libjvm.dylib+0x49bf1d], sa_mask[0]=11111111011111110111111111111111, sa_flags=SA_RESTART|SA_SIGINFO Start time: 2019-11-04 19:57:01Z End time: 2019-11-04 19:57:02Z Duration: 1.803 seconds This crash doesn't happen "in the wild" so it is unlikely to get fixed anytime soon. As a workaround for our CI, we will use AzDO's [`retryCountOnTaskFailure` parameter][0] to automatically re-run a native test suite once if it fails. This should reduce the need for us to re-run failed pipelines for flaky tests. Note that AzDO's implementation isn't perfect; the task will succeed if there are no failures on retry, but the original failures will still be listed in the log. 🤷‍♂️ [0]: https://learn.microsoft.com/en-us/azure/devops/release-notes/2021/pipelines/sprint-195-update#automatic-retries-for-a-task
diff --git a/build-tools/automation/templates/core-tests.yaml b/build-tools/automation/templates/core-tests.yaml
@@ -84,6 +84,7 @@ steps:
     testRunTitle: Java.Interop (netstandard2.0 - ${{ parameters.platformName }})
     arguments: bin/Test$(Build.Configuration)/Java.Interop-Tests.dll
   continueOnError: true
+  retryCountOnTaskFailure: 1
 
 - task: DotNetCoreCLI@2
   displayName: 'Tests: Java.Interop'
@@ -93,6 +94,7 @@ steps:
     testRunTitle: Java.Interop ($(DotNetTargetFramework) - ${{ parameters.platformName }})
     arguments: bin/Test$(Build.Configuration)$(NetCoreTargetFrameworkPathSuffix)/Java.Interop-Tests.dll
   continueOnError: true
+  retryCountOnTaskFailure: 1
 
 - task: DotNetCoreCLI@2
   displayName: 'Tests: Java.Interop.Dynamic'
@@ -102,6 +104,7 @@ steps:
     testRunTitle: Java.Interop.Dynamic (${{ parameters.platformName }})
     arguments: bin/Test$(Build.Configuration)$(NetCoreTargetFrameworkPathSuffix)/Java.Interop.Dynamic-Tests.dll
   continueOnError: true
+  retryCountOnTaskFailure: 1
 
 - task: DotNetCoreCLI@2
   displayName: 'Tests: Java.Interop.Export'
@@ -111,6 +114,7 @@ steps:
     testRunTitle: Java.Interop.Export (${{ parameters.platformName }})
     arguments: bin/Test$(Build.Configuration)$(NetCoreTargetFrameworkPathSuffix)/Java.Interop.Export-Tests.dll
   continueOnError: true
+  retryCountOnTaskFailure: 1
 
 - task: DotNetCoreCLI@2
   displayName: 'jnimarshalmethod-gen Java.Interop.Export-Tests.dll'
@@ -120,6 +124,7 @@ steps:
     custom: bin/$(Build.Configuration)$(NetCoreTargetFrameworkPathSuffix)/jnimarshalmethod-gen.dll
     arguments: bin/Test$(Build.Configuration)$(NetCoreTargetFrameworkPathSuffix)/Java.Interop.Export-Tests.dll -v -v --keeptemp -o bin/Test$(Build.Configuration)$(NetCoreTargetFrameworkPathSuffix)
   continueOnError: true
+  retryCountOnTaskFailure: 1
 
 - task: DotNetCoreCLI@2
   displayName: 'Tests: Java.Interop.Export w/ jnimarshalmethod-gen!'
@@ -129,6 +134,7 @@ steps:
     testRunTitle: Java.Interop.Export (jnimarshalmethod-gen + ${{ parameters.platformName }})
     arguments: bin/Test$(Build.Configuration)$(NetCoreTargetFrameworkPathSuffix)/Java.Interop.Export-Tests.dll
   continueOnError: true
+  retryCountOnTaskFailure: 1
 
 - task: DotNetCoreCLI@2
   displayName: 'Tests: Java.Interop-Performance-net472'
@@ -138,6 +144,7 @@ steps:
     testRunTitle: Java.Interop-Performance (net472 - ${{ parameters.platformName }})
     arguments: bin/Test$(Build.Configuration)/Java.Interop-PerformanceTests.dll
   continueOnError: true
+  retryCountOnTaskFailure: 1
 
 - task: DotNetCoreCLI@2
   displayName: 'Tests: Java.Interop-Performance-$(DotNetTargetFramework)'
@@ -147,6 +154,7 @@ steps:
     testRunTitle: Java.Interop-Performance ($(DotNetTargetFramework) - ${{ parameters.platformName }})
     arguments: bin/Test$(Build.Configuration)$(NetCoreTargetFrameworkPathSuffix)/Java.Interop-PerformanceTests.dll
   continueOnError: true
+  retryCountOnTaskFailure: 1
 
 - task: DotNetCoreCLI@2
   displayName: 'Tests: Java.Base'
@@ -156,6 +164,7 @@ steps:
     testRunTitle: Java.Base ($(DotNetTargetFramework) - ${{ parameters.platformName }})
     arguments: bin/Test$(Build.Configuration)$(NetCoreTargetFrameworkPathSuffix)/Java.Base-Tests.dll
   continueOnError: true
+  retryCountOnTaskFailure: 1
 
 - task: DotNetCoreCLI@2
   displayName: 'Tests: java-source-utils'
