From 9d454678f94dcdc2a90888770fa30c7940455372 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Sun, 22 Oct 2023 15:19:01 +0900 Subject: [PATCH 01/31] Add BreakDurationGenerator --- .../CircuitBreakerStrategyOptions.TResult.cs | 8 +++++ .../Controller/AdvancedCircuitBehavior.cs | 1 + .../Controller/CircuitBehavior.cs | 1 + .../Controller/CircuitStateController.cs | 34 ++++++++++++++++++- .../CircuitBreaker/Health/HealthInfo.cs | 6 ++-- 5 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs index 47cc7a9ecfb..ffa78a2fe2f 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs @@ -69,6 +69,14 @@ public class CircuitBreakerStrategyOptions : ResilienceStrategyOptions public TimeSpan BreakDuration { get; set; } = CircuitBreakerConstants.DefaultBreakDuration; #pragma warning restore + /// + /// Gets or sets the function responsible for dynamically generating the break duration based on the health failure count. + /// + /// + /// A function that takes an integer representing the health failure count and returns a TimeSpan indicating the break duration. + /// + public Func? BreakDurationGenerator { get; set; } + /// /// Gets or sets a predicate that determines whether the outcome should be handled by the circuit breaker. /// diff --git a/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs index 8cfc1cb472d..db15ae7d357 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs @@ -44,5 +44,6 @@ public override void OnActionFailure(CircuitState currentState, out bool shouldB } public override void OnCircuitClosed() => _metrics.Reset(); + public override int FailureCount => _metrics.GetHealthInfo().FailureCount; } diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs index d1e1f2ee710..2ed1fa860bf 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs @@ -10,4 +10,5 @@ internal abstract class CircuitBehavior public abstract void OnActionFailure(CircuitState currentState, out bool shouldBreak); public abstract void OnCircuitClosed(); + public abstract int FailureCount { get; } } diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs index 37c0c70db7f..cb89860fa3f 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs @@ -16,6 +16,7 @@ internal sealed class CircuitStateController : IDisposable private readonly ResilienceStrategyTelemetry _telemetry; private readonly CircuitBehavior _behavior; private readonly TimeSpan _breakDuration; + private readonly Func _breakDurationGenerator; private DateTimeOffset _blockedUntil; private CircuitState _circuitState = CircuitState.Closed; private Outcome? _lastOutcome; @@ -40,6 +41,28 @@ public CircuitStateController( _telemetry = telemetry; } + public CircuitStateController( + TimeSpan breakDuration, + Func breakDurationGenerator, + Func, ValueTask>? onOpened, + Func, ValueTask>? onClosed, + Func? onHalfOpen, + CircuitBehavior behavior, + TimeProvider timeProvider, + ResilienceStrategyTelemetry telemetry) + { + + _breakDuration = breakDuration; + _breakDurationGenerator = breakDurationGenerator; + _onOpened = onOpened; + _onClosed = onClosed; + _onHalfOpen = onHalfOpen; + _behavior = behavior; + _timeProvider = timeProvider; + _telemetry = telemetry; + } + + public CircuitState CircuitState { get @@ -314,7 +337,16 @@ private void OpenCircuitFor_NeedsLock(Outcome outcome, TimeSpan breakDuration scheduledTask = null; var utcNow = _timeProvider.GetUtcNow(); - _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; + if (_breakDurationGenerator is not null && _behavior.FailureCount > 0) + { + var generatedBreakDuration = _breakDurationGenerator(_behavior.FailureCount); + _blockedUntil = IsDateTimeOverflow(utcNow, generatedBreakDuration) ? DateTimeOffset.MaxValue : utcNow + generatedBreakDuration; + } + else + { + _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; + } + var transitionedState = _circuitState; _circuitState = CircuitState.Open; diff --git a/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs b/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs index 0e014526b54..43eac6cb945 100644 --- a/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs +++ b/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs @@ -1,15 +1,15 @@ namespace Polly.CircuitBreaker.Health; -internal readonly record struct HealthInfo(int Throughput, double FailureRate) +internal readonly record struct HealthInfo(int Throughput, double FailureRate,int FailureCount = default(int)) { public static HealthInfo Create(int successes, int failures) { var total = successes + failures; if (total == 0) { - return new HealthInfo(0, 0); + return new HealthInfo(0, 0, failures); } - return new(total, failures / (double)total); + return new(total, failures / (double)total,failures); } } From 488091122e8624c3ff0445358e40acf0351171c0 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Sun, 22 Oct 2023 22:51:38 +0900 Subject: [PATCH 02/31] Add Duration Generator UnitTest - CircuitStateControllerTest - RollingHealthMetrixTest --- .../Controller/CircuitStateControllerTests.cs | 49 +++++++++++++++++++ .../Health/RollingHealthMetricsTests.cs | 10 ++-- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs index 4c0784098a5..96ce0031703 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -303,6 +303,45 @@ public async Task OnActionFailureAsync_EnsureCorrectBehavior(CircuitState state, } } + [Fact] + public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() + { + // arrange + var result = new TimeSpan(0, 0, 10, 0); + using var controller = CreateController(new () + { + FailureRatio = 0, + MinimumThroughput = 0, + SamplingDuration = default, + BreakDuration = new TimeSpan(0,0,1,0), + BreakDurationGenerator = (failureCount) => result, + OnClosed = null, + OnOpened = null, + OnHalfOpened = null, + ManualControl = null, + StateProvider = null + }); + + await TransitionToState(controller, CircuitState.Closed); + var utcNow = DateTimeOffset.MaxValue - result; + + _timeProvider.SetUtcNow(utcNow); + _circuitBehavior.FailureCount.Returns(1); + _circuitBehavior.When(v => v.OnActionFailure(CircuitState.Closed, out Arg.Any())) + .Do(x => + { + x[1] = true; + }); + + // act + await controller.OnActionFailureAsync(Outcome.FromResult(99), ResilienceContextPool.Shared.Get()); + + // assert + var blockedTill = GetBlockedTill(controller); + + blockedTill.Should().Be(DateTimeOffset.MaxValue); + } + [InlineData(true)] [InlineData(false)] [Theory] @@ -466,4 +505,14 @@ private async Task OpenCircuit(CircuitStateController controller, Outcome CreateController(CircuitBreakerStrategyOptions options) => new( + options.BreakDuration, + options.BreakDurationGenerator, + options.OnOpened, + options.OnClosed, + options.OnHalfOpened, + _circuitBehavior, + _timeProvider, + TestUtilities.CreateResilienceTelemetry(_telemetryListener)); } diff --git a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs index 2c4d5d8005d..1e428f8f503 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs @@ -62,11 +62,11 @@ public void GetHealthInfo_EnsureWindowRespected() _timeProvider.Advance(TimeSpan.FromSeconds(2)); health.Add(metrics.GetHealthInfo()); - health[0].Should().Be(new HealthInfo(2, 0.5)); - health[1].Should().Be(new HealthInfo(4, 0.5)); - health[3].Should().Be(new HealthInfo(8, 0.25)); - health[4].Should().Be(new HealthInfo(8, 0.125)); - health[5].Should().Be(new HealthInfo(6, 0.0)); + health[0].Should().Be(new HealthInfo(2, 0.5, 1)); + health[1].Should().Be(new HealthInfo(4, 0.5, 2)); + health[3].Should().Be(new HealthInfo(8, 0.25, 2)); + health[4].Should().Be(new HealthInfo(8, 0.125,1)); + health[5].Should().Be(new HealthInfo(6, 0.0,0)); } [Fact] From 5752bb6742a10252423cb06375417b5ae6d59630 Mon Sep 17 00:00:00 2001 From: LeeJonghoon <34878017+atawLee@users.noreply.github.com> Date: Mon, 23 Oct 2023 23:50:04 +0900 Subject: [PATCH 03/31] Apply suggestions from code review apply code convention feedback Co-authored-by: Martin Costello --- .../CircuitBreakerStrategyOptions.TResult.cs | 4 ++-- src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs | 4 ++-- .../Controller/CircuitStateControllerTests.cs | 7 ++----- .../CircuitBreaker/Health/RollingHealthMetricsTests.cs | 4 ++-- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs index ffa78a2fe2f..4b2d2bec7ad 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs @@ -70,10 +70,10 @@ public class CircuitBreakerStrategyOptions : ResilienceStrategyOptions #pragma warning restore /// - /// Gets or sets the function responsible for dynamically generating the break duration based on the health failure count. + /// Gets or sets an optional delegate to use to dynamically generate the break duration based on the health failure count. /// /// - /// A function that takes an integer representing the health failure count and returns a TimeSpan indicating the break duration. + /// A delegate that takes an integer representing the health failure count and returns a indicating the break duration. /// public Func? BreakDurationGenerator { get; set; } diff --git a/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs b/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs index 43eac6cb945..337cfb7c90d 100644 --- a/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs +++ b/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs @@ -1,6 +1,6 @@ namespace Polly.CircuitBreaker.Health; -internal readonly record struct HealthInfo(int Throughput, double FailureRate,int FailureCount = default(int)) +internal readonly record struct HealthInfo(int Throughput, double FailureRate, int FailureCount = default(int)) { public static HealthInfo Create(int successes, int failures) { @@ -10,6 +10,6 @@ public static HealthInfo Create(int successes, int failures) return new HealthInfo(0, 0, failures); } - return new(total, failures / (double)total,failures); + return new(total, failures / (double)total, failures); } } diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs index 96ce0031703..a7349b0efdb 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -313,7 +313,7 @@ public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() FailureRatio = 0, MinimumThroughput = 0, SamplingDuration = default, - BreakDuration = new TimeSpan(0,0,1,0), + BreakDuration = TimeSpan.FromMinutes(1), BreakDurationGenerator = (failureCount) => result, OnClosed = null, OnOpened = null, @@ -328,10 +328,7 @@ public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() _timeProvider.SetUtcNow(utcNow); _circuitBehavior.FailureCount.Returns(1); _circuitBehavior.When(v => v.OnActionFailure(CircuitState.Closed, out Arg.Any())) - .Do(x => - { - x[1] = true; - }); + .Do(x => x[1] = true); // act await controller.OnActionFailureAsync(Outcome.FromResult(99), ResilienceContextPool.Shared.Get()); diff --git a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs index 1e428f8f503..43a5549c6a2 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs @@ -65,8 +65,8 @@ public void GetHealthInfo_EnsureWindowRespected() health[0].Should().Be(new HealthInfo(2, 0.5, 1)); health[1].Should().Be(new HealthInfo(4, 0.5, 2)); health[3].Should().Be(new HealthInfo(8, 0.25, 2)); - health[4].Should().Be(new HealthInfo(8, 0.125,1)); - health[5].Should().Be(new HealthInfo(6, 0.0,0)); + health[4].Should().Be(new HealthInfo(8, 0.125, 1)); + health[5].Should().Be(new HealthInfo(6, 0.0, 0)); } [Fact] From 22e25907ed248472d5153bcc6097ec1bb3c2acd5 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Thu, 26 Oct 2023 23:49:13 +0900 Subject: [PATCH 04/31] Add BreakDurationGeneratorArguments --- .../BreakDurationGeneratorArguments.cs | 23 ++++++++++++ .../CircuitBreakerStrategyOptions.TResult.cs | 3 +- .../Controller/AdvancedCircuitBehavior.cs | 1 + .../Controller/CircuitBehavior.cs | 1 + .../Controller/CircuitStateController.cs | 36 ++++--------------- .../Controller/CircuitStateControllerTests.cs | 5 +-- 6 files changed, 36 insertions(+), 33 deletions(-) create mode 100644 src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs diff --git a/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs new file mode 100644 index 00000000000..1a23aa4ccde --- /dev/null +++ b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; +using System; +using System.Runtime.CompilerServices; + +namespace Polly.CircuitBreaker; + +/// +/// Represents arguments used to generate a dynamic break duration for a circuit breaker. +/// +public class BreakDurationGeneratorArguments +{ + public BreakDurationGeneratorArguments( + double failureRate, + int failureCount) + { + FailureRate = failureRate; + FailureCount = failureCount; + } + + public double FailureRate { get; set; } + + public int FailureCount { get; set; } +} diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs index 4b2d2bec7ad..f1ae424d51f 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs @@ -1,3 +1,4 @@ +using System; using System.ComponentModel.DataAnnotations; namespace Polly.CircuitBreaker; @@ -75,7 +76,7 @@ public class CircuitBreakerStrategyOptions : ResilienceStrategyOptions /// /// A delegate that takes an integer representing the health failure count and returns a indicating the break duration. /// - public Func? BreakDurationGenerator { get; set; } + public Func? BreakDurationGenerator { get; set; } /// /// Gets or sets a predicate that determines whether the outcome should be handled by the circuit breaker. diff --git a/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs index db15ae7d357..3b931715771 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs @@ -45,5 +45,6 @@ public override void OnActionFailure(CircuitState currentState, out bool shouldB public override void OnCircuitClosed() => _metrics.Reset(); public override int FailureCount => _metrics.GetHealthInfo().FailureCount; + public override double FailureRate => _metrics.GetHealthInfo().FailureRate; } diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs index 2ed1fa860bf..a585756f73a 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs @@ -11,4 +11,5 @@ internal abstract class CircuitBehavior public abstract void OnCircuitClosed(); public abstract int FailureCount { get; } + public abstract double FailureRate { get; } } diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs index cb89860fa3f..5eb834cf9f7 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs @@ -16,7 +16,7 @@ internal sealed class CircuitStateController : IDisposable private readonly ResilienceStrategyTelemetry _telemetry; private readonly CircuitBehavior _behavior; private readonly TimeSpan _breakDuration; - private readonly Func _breakDurationGenerator; + private readonly Func? _breakDurationGenerator; private DateTimeOffset _blockedUntil; private CircuitState _circuitState = CircuitState.Closed; private Outcome? _lastOutcome; @@ -30,7 +30,8 @@ public CircuitStateController( Func? onHalfOpen, CircuitBehavior behavior, TimeProvider timeProvider, - ResilienceStrategyTelemetry telemetry) + ResilienceStrategyTelemetry telemetry, + Func? breakDurationGenerator = null) { _breakDuration = breakDuration; _onOpened = onOpened; @@ -39,30 +40,9 @@ public CircuitStateController( _behavior = behavior; _timeProvider = timeProvider; _telemetry = telemetry; - } - - public CircuitStateController( - TimeSpan breakDuration, - Func breakDurationGenerator, - Func, ValueTask>? onOpened, - Func, ValueTask>? onClosed, - Func? onHalfOpen, - CircuitBehavior behavior, - TimeProvider timeProvider, - ResilienceStrategyTelemetry telemetry) - { - - _breakDuration = breakDuration; _breakDurationGenerator = breakDurationGenerator; - _onOpened = onOpened; - _onClosed = onClosed; - _onHalfOpen = onHalfOpen; - _behavior = behavior; - _timeProvider = timeProvider; - _telemetry = telemetry; } - public CircuitState CircuitState { get @@ -339,15 +319,11 @@ private void OpenCircuitFor_NeedsLock(Outcome outcome, TimeSpan breakDuration if (_breakDurationGenerator is not null && _behavior.FailureCount > 0) { - var generatedBreakDuration = _breakDurationGenerator(_behavior.FailureCount); - _blockedUntil = IsDateTimeOverflow(utcNow, generatedBreakDuration) ? DateTimeOffset.MaxValue : utcNow + generatedBreakDuration; - } - else - { - _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; + breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount)); } - + _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; + var transitionedState = _circuitState; _circuitState = CircuitState.Open; diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs index a7349b0efdb..03cf654a24f 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -505,11 +505,12 @@ private async Task OpenCircuit(CircuitStateController controller, Outcome CreateController(CircuitBreakerStrategyOptions options) => new( options.BreakDuration, - options.BreakDurationGenerator, options.OnOpened, options.OnClosed, options.OnHalfOpened, _circuitBehavior, _timeProvider, - TestUtilities.CreateResilienceTelemetry(_telemetryListener)); + TestUtilities.CreateResilienceTelemetry(_telemetryListener), + options.BreakDurationGenerator + ); } From 29942beb0178718d420e6355718babfb659a24bb Mon Sep 17 00:00:00 2001 From: LeeJonghoon <34878017+atawLee@users.noreply.github.com> Date: Fri, 27 Oct 2023 00:26:58 +0900 Subject: [PATCH 05/31] Update circuit-breaker.md Add CircuitBreaker BreakDurationGenerator Document Update Usage Update Anti-pattern --- docs/strategies/circuit-breaker.md | 46 +++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/strategies/circuit-breaker.md b/docs/strategies/circuit-breaker.md index 50ff7a5b7e6..b97ccc0c74b 100644 --- a/docs/strategies/circuit-breaker.md +++ b/docs/strategies/circuit-breaker.md @@ -37,6 +37,32 @@ new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOpti ShouldHandle = new PredicateBuilder().Handle() }); +// Add circuit breaker with customized options: +// +// The circuit will break if more than 50% of actions result in handled exceptions, +// within any 10-second sampling duration, and at least 8 actions are processed. +new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions +{ + FailureRatio = 0.5, + SamplingDuration = TimeSpan.FromSeconds(10), + MinimumThroughput = 8, + BreakDuration = TimeSpan.FromSeconds(30), + ShouldHandle = new PredicateBuilder().Handle() +}); + +// Adds a circuit breaker with dynamic break duration: +// +// Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. +// The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. +new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions +{ + FailureRatio = 0.5, + SamplingDuration = TimeSpan.FromSeconds(10), + MinimumThroughput = 8, + ShouldHandle = new PredicateBuilder().Handle(), + BreakDurationGenerator = (args) => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)) +}); + // Handle specific failed results for HttpResponseMessage: new ResiliencePipelineBuilder() .AddCircuitBreaker(new CircuitBreakerStrategyOptions @@ -466,7 +492,24 @@ circuitBreaker = new ResiliencePipelineBuilder() ✅ DO -The `CircuitBreakerStrategyOptions` currently do not support defining break durations dynamically. This may be re-evaluated in the future. For now, refer to the first example for a potential workaround. However, please use it with caution. +Use the 'BreakDurationGenerator' to dynamically define the break duration: +```cs +// Adds a circuit breaker with dynamic break duration: +// +// Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. +// The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. +new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions +{ + FailureRatio = 0.5, + SamplingDuration = TimeSpan.FromSeconds(10), + MinimumThroughput = 8, + ShouldHandle = new PredicateBuilder().Handle(), + BreakDurationGenerator = (args) => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)) +}); +``` +**Reasoning**: + +- Using a BreakDurationGenerator like this, you can dynamically adjust the break duration with each failure. ### 3 - Wrapping each endpoint with a circuit breaker @@ -521,6 +564,7 @@ public Downstream1Client( ... } ``` +Using a BreakDurationGenerator like this, you can dynamically adjust the break duration with each failure. **Reasoning**: From 030b2345b2f3b784a07db986ea23dd6c227abd6b Mon Sep 17 00:00:00 2001 From: LeeJonghoon <34878017+atawLee@users.noreply.github.com> Date: Fri, 27 Oct 2023 00:26:58 +0900 Subject: [PATCH 06/31] Update circuit-breaker.md Add CircuitBreaker BreakDurationGenerator Document Update Usage Update Anti-pattern --- docs/strategies/circuit-breaker.md | 34 +++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/strategies/circuit-breaker.md b/docs/strategies/circuit-breaker.md index 50ff7a5b7e6..174f4dd05c4 100644 --- a/docs/strategies/circuit-breaker.md +++ b/docs/strategies/circuit-breaker.md @@ -37,6 +37,20 @@ new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOpti ShouldHandle = new PredicateBuilder().Handle() }); + +// Adds a circuit breaker with dynamic break duration: +// +// Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. +// The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. +new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions +{ + FailureRatio = 0.5, + SamplingDuration = TimeSpan.FromSeconds(10), + MinimumThroughput = 8, + ShouldHandle = new PredicateBuilder().Handle(), + BreakDurationGenerator = (args) => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)) +}); + // Handle specific failed results for HttpResponseMessage: new ResiliencePipelineBuilder() .AddCircuitBreaker(new CircuitBreakerStrategyOptions @@ -466,7 +480,24 @@ circuitBreaker = new ResiliencePipelineBuilder() ✅ DO -The `CircuitBreakerStrategyOptions` currently do not support defining break durations dynamically. This may be re-evaluated in the future. For now, refer to the first example for a potential workaround. However, please use it with caution. +Use the 'BreakDurationGenerator' to dynamically define the break duration: +```cs +// Adds a circuit breaker with dynamic break duration: +// +// Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. +// The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. +new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions +{ + FailureRatio = 0.5, + SamplingDuration = TimeSpan.FromSeconds(10), + MinimumThroughput = 8, + ShouldHandle = new PredicateBuilder().Handle(), + BreakDurationGenerator = (args) => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)) +}); +``` +**Reasoning**: + +- Using a BreakDurationGenerator like this, you can dynamically adjust the break duration with each failure. ### 3 - Wrapping each endpoint with a circuit breaker @@ -521,6 +552,7 @@ public Downstream1Client( ... } ``` +Using a BreakDurationGenerator like this, you can dynamically adjust the break duration with each failure. **Reasoning**: From 8f9a59432e1b7107dae4a4f3c6d6920c4559a5ae Mon Sep 17 00:00:00 2001 From: jonghoon Date: Sun, 22 Oct 2023 15:19:01 +0900 Subject: [PATCH 07/31] Add BreakDurationGenerator --- .../CircuitBreakerStrategyOptions.TResult.cs | 8 +++++ .../Controller/AdvancedCircuitBehavior.cs | 1 + .../Controller/CircuitBehavior.cs | 1 + .../Controller/CircuitStateController.cs | 34 ++++++++++++++++++- .../CircuitBreaker/Health/HealthInfo.cs | 6 ++-- 5 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs index 47cc7a9ecfb..ffa78a2fe2f 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs @@ -69,6 +69,14 @@ public class CircuitBreakerStrategyOptions : ResilienceStrategyOptions public TimeSpan BreakDuration { get; set; } = CircuitBreakerConstants.DefaultBreakDuration; #pragma warning restore + /// + /// Gets or sets the function responsible for dynamically generating the break duration based on the health failure count. + /// + /// + /// A function that takes an integer representing the health failure count and returns a TimeSpan indicating the break duration. + /// + public Func? BreakDurationGenerator { get; set; } + /// /// Gets or sets a predicate that determines whether the outcome should be handled by the circuit breaker. /// diff --git a/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs index 8cfc1cb472d..db15ae7d357 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs @@ -44,5 +44,6 @@ public override void OnActionFailure(CircuitState currentState, out bool shouldB } public override void OnCircuitClosed() => _metrics.Reset(); + public override int FailureCount => _metrics.GetHealthInfo().FailureCount; } diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs index d1e1f2ee710..2ed1fa860bf 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs @@ -10,4 +10,5 @@ internal abstract class CircuitBehavior public abstract void OnActionFailure(CircuitState currentState, out bool shouldBreak); public abstract void OnCircuitClosed(); + public abstract int FailureCount { get; } } diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs index 37c0c70db7f..cb89860fa3f 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs @@ -16,6 +16,7 @@ internal sealed class CircuitStateController : IDisposable private readonly ResilienceStrategyTelemetry _telemetry; private readonly CircuitBehavior _behavior; private readonly TimeSpan _breakDuration; + private readonly Func _breakDurationGenerator; private DateTimeOffset _blockedUntil; private CircuitState _circuitState = CircuitState.Closed; private Outcome? _lastOutcome; @@ -40,6 +41,28 @@ public CircuitStateController( _telemetry = telemetry; } + public CircuitStateController( + TimeSpan breakDuration, + Func breakDurationGenerator, + Func, ValueTask>? onOpened, + Func, ValueTask>? onClosed, + Func? onHalfOpen, + CircuitBehavior behavior, + TimeProvider timeProvider, + ResilienceStrategyTelemetry telemetry) + { + + _breakDuration = breakDuration; + _breakDurationGenerator = breakDurationGenerator; + _onOpened = onOpened; + _onClosed = onClosed; + _onHalfOpen = onHalfOpen; + _behavior = behavior; + _timeProvider = timeProvider; + _telemetry = telemetry; + } + + public CircuitState CircuitState { get @@ -314,7 +337,16 @@ private void OpenCircuitFor_NeedsLock(Outcome outcome, TimeSpan breakDuration scheduledTask = null; var utcNow = _timeProvider.GetUtcNow(); - _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; + if (_breakDurationGenerator is not null && _behavior.FailureCount > 0) + { + var generatedBreakDuration = _breakDurationGenerator(_behavior.FailureCount); + _blockedUntil = IsDateTimeOverflow(utcNow, generatedBreakDuration) ? DateTimeOffset.MaxValue : utcNow + generatedBreakDuration; + } + else + { + _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; + } + var transitionedState = _circuitState; _circuitState = CircuitState.Open; diff --git a/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs b/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs index 0e014526b54..43eac6cb945 100644 --- a/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs +++ b/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs @@ -1,15 +1,15 @@ namespace Polly.CircuitBreaker.Health; -internal readonly record struct HealthInfo(int Throughput, double FailureRate) +internal readonly record struct HealthInfo(int Throughput, double FailureRate,int FailureCount = default(int)) { public static HealthInfo Create(int successes, int failures) { var total = successes + failures; if (total == 0) { - return new HealthInfo(0, 0); + return new HealthInfo(0, 0, failures); } - return new(total, failures / (double)total); + return new(total, failures / (double)total,failures); } } From b9023aa2d95003347d07384dfdb9ea9899819a52 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Sun, 22 Oct 2023 22:51:38 +0900 Subject: [PATCH 08/31] Add Duration Generator UnitTest - CircuitStateControllerTest - RollingHealthMetrixTest --- .../Controller/CircuitStateControllerTests.cs | 49 +++++++++++++++++++ .../Health/RollingHealthMetricsTests.cs | 10 ++-- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs index 4c0784098a5..96ce0031703 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -303,6 +303,45 @@ public async Task OnActionFailureAsync_EnsureCorrectBehavior(CircuitState state, } } + [Fact] + public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() + { + // arrange + var result = new TimeSpan(0, 0, 10, 0); + using var controller = CreateController(new () + { + FailureRatio = 0, + MinimumThroughput = 0, + SamplingDuration = default, + BreakDuration = new TimeSpan(0,0,1,0), + BreakDurationGenerator = (failureCount) => result, + OnClosed = null, + OnOpened = null, + OnHalfOpened = null, + ManualControl = null, + StateProvider = null + }); + + await TransitionToState(controller, CircuitState.Closed); + var utcNow = DateTimeOffset.MaxValue - result; + + _timeProvider.SetUtcNow(utcNow); + _circuitBehavior.FailureCount.Returns(1); + _circuitBehavior.When(v => v.OnActionFailure(CircuitState.Closed, out Arg.Any())) + .Do(x => + { + x[1] = true; + }); + + // act + await controller.OnActionFailureAsync(Outcome.FromResult(99), ResilienceContextPool.Shared.Get()); + + // assert + var blockedTill = GetBlockedTill(controller); + + blockedTill.Should().Be(DateTimeOffset.MaxValue); + } + [InlineData(true)] [InlineData(false)] [Theory] @@ -466,4 +505,14 @@ private async Task OpenCircuit(CircuitStateController controller, Outcome CreateController(CircuitBreakerStrategyOptions options) => new( + options.BreakDuration, + options.BreakDurationGenerator, + options.OnOpened, + options.OnClosed, + options.OnHalfOpened, + _circuitBehavior, + _timeProvider, + TestUtilities.CreateResilienceTelemetry(_telemetryListener)); } diff --git a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs index 2c4d5d8005d..1e428f8f503 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs @@ -62,11 +62,11 @@ public void GetHealthInfo_EnsureWindowRespected() _timeProvider.Advance(TimeSpan.FromSeconds(2)); health.Add(metrics.GetHealthInfo()); - health[0].Should().Be(new HealthInfo(2, 0.5)); - health[1].Should().Be(new HealthInfo(4, 0.5)); - health[3].Should().Be(new HealthInfo(8, 0.25)); - health[4].Should().Be(new HealthInfo(8, 0.125)); - health[5].Should().Be(new HealthInfo(6, 0.0)); + health[0].Should().Be(new HealthInfo(2, 0.5, 1)); + health[1].Should().Be(new HealthInfo(4, 0.5, 2)); + health[3].Should().Be(new HealthInfo(8, 0.25, 2)); + health[4].Should().Be(new HealthInfo(8, 0.125,1)); + health[5].Should().Be(new HealthInfo(6, 0.0,0)); } [Fact] From 31ff325b03a89b65a704b4d1f20f37e8a773e896 Mon Sep 17 00:00:00 2001 From: LeeJonghoon <34878017+atawLee@users.noreply.github.com> Date: Mon, 23 Oct 2023 23:50:04 +0900 Subject: [PATCH 09/31] Apply suggestions from code review apply code convention feedback Co-authored-by: Martin Costello --- .../CircuitBreakerStrategyOptions.TResult.cs | 4 ++-- src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs | 4 ++-- .../Controller/CircuitStateControllerTests.cs | 7 ++----- .../CircuitBreaker/Health/RollingHealthMetricsTests.cs | 4 ++-- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs index ffa78a2fe2f..4b2d2bec7ad 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs @@ -70,10 +70,10 @@ public class CircuitBreakerStrategyOptions : ResilienceStrategyOptions #pragma warning restore /// - /// Gets or sets the function responsible for dynamically generating the break duration based on the health failure count. + /// Gets or sets an optional delegate to use to dynamically generate the break duration based on the health failure count. /// /// - /// A function that takes an integer representing the health failure count and returns a TimeSpan indicating the break duration. + /// A delegate that takes an integer representing the health failure count and returns a indicating the break duration. /// public Func? BreakDurationGenerator { get; set; } diff --git a/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs b/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs index 43eac6cb945..337cfb7c90d 100644 --- a/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs +++ b/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs @@ -1,6 +1,6 @@ namespace Polly.CircuitBreaker.Health; -internal readonly record struct HealthInfo(int Throughput, double FailureRate,int FailureCount = default(int)) +internal readonly record struct HealthInfo(int Throughput, double FailureRate, int FailureCount = default(int)) { public static HealthInfo Create(int successes, int failures) { @@ -10,6 +10,6 @@ public static HealthInfo Create(int successes, int failures) return new HealthInfo(0, 0, failures); } - return new(total, failures / (double)total,failures); + return new(total, failures / (double)total, failures); } } diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs index 96ce0031703..a7349b0efdb 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -313,7 +313,7 @@ public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() FailureRatio = 0, MinimumThroughput = 0, SamplingDuration = default, - BreakDuration = new TimeSpan(0,0,1,0), + BreakDuration = TimeSpan.FromMinutes(1), BreakDurationGenerator = (failureCount) => result, OnClosed = null, OnOpened = null, @@ -328,10 +328,7 @@ public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() _timeProvider.SetUtcNow(utcNow); _circuitBehavior.FailureCount.Returns(1); _circuitBehavior.When(v => v.OnActionFailure(CircuitState.Closed, out Arg.Any())) - .Do(x => - { - x[1] = true; - }); + .Do(x => x[1] = true); // act await controller.OnActionFailureAsync(Outcome.FromResult(99), ResilienceContextPool.Shared.Get()); diff --git a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs index 1e428f8f503..43a5549c6a2 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs @@ -65,8 +65,8 @@ public void GetHealthInfo_EnsureWindowRespected() health[0].Should().Be(new HealthInfo(2, 0.5, 1)); health[1].Should().Be(new HealthInfo(4, 0.5, 2)); health[3].Should().Be(new HealthInfo(8, 0.25, 2)); - health[4].Should().Be(new HealthInfo(8, 0.125,1)); - health[5].Should().Be(new HealthInfo(6, 0.0,0)); + health[4].Should().Be(new HealthInfo(8, 0.125, 1)); + health[5].Should().Be(new HealthInfo(6, 0.0, 0)); } [Fact] From 2faadbdb8a47ee9f1e47335a6f206c8554cd1a29 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Thu, 26 Oct 2023 23:49:13 +0900 Subject: [PATCH 10/31] Add BreakDurationGeneratorArguments --- .../BreakDurationGeneratorArguments.cs | 23 ++++++++++++ .../CircuitBreakerStrategyOptions.TResult.cs | 3 +- .../Controller/AdvancedCircuitBehavior.cs | 1 + .../Controller/CircuitBehavior.cs | 1 + .../Controller/CircuitStateController.cs | 36 ++++--------------- .../Controller/CircuitStateControllerTests.cs | 5 +-- 6 files changed, 36 insertions(+), 33 deletions(-) create mode 100644 src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs diff --git a/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs new file mode 100644 index 00000000000..1a23aa4ccde --- /dev/null +++ b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; +using System; +using System.Runtime.CompilerServices; + +namespace Polly.CircuitBreaker; + +/// +/// Represents arguments used to generate a dynamic break duration for a circuit breaker. +/// +public class BreakDurationGeneratorArguments +{ + public BreakDurationGeneratorArguments( + double failureRate, + int failureCount) + { + FailureRate = failureRate; + FailureCount = failureCount; + } + + public double FailureRate { get; set; } + + public int FailureCount { get; set; } +} diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs index 4b2d2bec7ad..f1ae424d51f 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs @@ -1,3 +1,4 @@ +using System; using System.ComponentModel.DataAnnotations; namespace Polly.CircuitBreaker; @@ -75,7 +76,7 @@ public class CircuitBreakerStrategyOptions : ResilienceStrategyOptions /// /// A delegate that takes an integer representing the health failure count and returns a indicating the break duration. /// - public Func? BreakDurationGenerator { get; set; } + public Func? BreakDurationGenerator { get; set; } /// /// Gets or sets a predicate that determines whether the outcome should be handled by the circuit breaker. diff --git a/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs index db15ae7d357..3b931715771 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs @@ -45,5 +45,6 @@ public override void OnActionFailure(CircuitState currentState, out bool shouldB public override void OnCircuitClosed() => _metrics.Reset(); public override int FailureCount => _metrics.GetHealthInfo().FailureCount; + public override double FailureRate => _metrics.GetHealthInfo().FailureRate; } diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs index 2ed1fa860bf..a585756f73a 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs @@ -11,4 +11,5 @@ internal abstract class CircuitBehavior public abstract void OnCircuitClosed(); public abstract int FailureCount { get; } + public abstract double FailureRate { get; } } diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs index cb89860fa3f..5eb834cf9f7 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs @@ -16,7 +16,7 @@ internal sealed class CircuitStateController : IDisposable private readonly ResilienceStrategyTelemetry _telemetry; private readonly CircuitBehavior _behavior; private readonly TimeSpan _breakDuration; - private readonly Func _breakDurationGenerator; + private readonly Func? _breakDurationGenerator; private DateTimeOffset _blockedUntil; private CircuitState _circuitState = CircuitState.Closed; private Outcome? _lastOutcome; @@ -30,7 +30,8 @@ public CircuitStateController( Func? onHalfOpen, CircuitBehavior behavior, TimeProvider timeProvider, - ResilienceStrategyTelemetry telemetry) + ResilienceStrategyTelemetry telemetry, + Func? breakDurationGenerator = null) { _breakDuration = breakDuration; _onOpened = onOpened; @@ -39,30 +40,9 @@ public CircuitStateController( _behavior = behavior; _timeProvider = timeProvider; _telemetry = telemetry; - } - - public CircuitStateController( - TimeSpan breakDuration, - Func breakDurationGenerator, - Func, ValueTask>? onOpened, - Func, ValueTask>? onClosed, - Func? onHalfOpen, - CircuitBehavior behavior, - TimeProvider timeProvider, - ResilienceStrategyTelemetry telemetry) - { - - _breakDuration = breakDuration; _breakDurationGenerator = breakDurationGenerator; - _onOpened = onOpened; - _onClosed = onClosed; - _onHalfOpen = onHalfOpen; - _behavior = behavior; - _timeProvider = timeProvider; - _telemetry = telemetry; } - public CircuitState CircuitState { get @@ -339,15 +319,11 @@ private void OpenCircuitFor_NeedsLock(Outcome outcome, TimeSpan breakDuration if (_breakDurationGenerator is not null && _behavior.FailureCount > 0) { - var generatedBreakDuration = _breakDurationGenerator(_behavior.FailureCount); - _blockedUntil = IsDateTimeOverflow(utcNow, generatedBreakDuration) ? DateTimeOffset.MaxValue : utcNow + generatedBreakDuration; - } - else - { - _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; + breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount)); } - + _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; + var transitionedState = _circuitState; _circuitState = CircuitState.Open; diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs index a7349b0efdb..03cf654a24f 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -505,11 +505,12 @@ private async Task OpenCircuit(CircuitStateController controller, Outcome CreateController(CircuitBreakerStrategyOptions options) => new( options.BreakDuration, - options.BreakDurationGenerator, options.OnOpened, options.OnClosed, options.OnHalfOpened, _circuitBehavior, _timeProvider, - TestUtilities.CreateResilienceTelemetry(_telemetryListener)); + TestUtilities.CreateResilienceTelemetry(_telemetryListener), + options.BreakDurationGenerator + ); } From 491c99ff0885189f4e60d0056cc9d176f2fc3ec1 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Fri, 27 Oct 2023 00:36:25 +0900 Subject: [PATCH 11/31] Update document circuit-breaker.md --- docs/strategies/circuit-breaker.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/strategies/circuit-breaker.md b/docs/strategies/circuit-breaker.md index 76267e88b1a..d746cd487a4 100644 --- a/docs/strategies/circuit-breaker.md +++ b/docs/strategies/circuit-breaker.md @@ -37,6 +37,19 @@ new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOpti ShouldHandle = new PredicateBuilder().Handle() }); +// Adds a circuit breaker with dynamic break duration: +// +// Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. +// The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. +new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions +{ + FailureRatio = 0.5, + SamplingDuration = TimeSpan.FromSeconds(10), + MinimumThroughput = 8, + ShouldHandle = new PredicateBuilder().Handle(), + BreakDurationGenerator = (args) => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)) +}); + // Handle specific failed results for HttpResponseMessage: new ResiliencePipelineBuilder() .AddCircuitBreaker(new CircuitBreakerStrategyOptions From 9d2ae692eca5e02244eced6df80604c0bc4db972 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Fri, 27 Oct 2023 00:39:08 +0900 Subject: [PATCH 12/31] fix document --- docs/strategies/circuit-breaker.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/strategies/circuit-breaker.md b/docs/strategies/circuit-breaker.md index d746cd487a4..e71e7012045 100644 --- a/docs/strategies/circuit-breaker.md +++ b/docs/strategies/circuit-breaker.md @@ -551,7 +551,6 @@ public Downstream1Client( ... } ``` -Using a BreakDurationGenerator like this, you can dynamically adjust the break duration with each failure. **Reasoning**: From b6a8948f83775f3611e6ad30c473bcd408bd175a Mon Sep 17 00:00:00 2001 From: jonghoon Date: Sat, 28 Oct 2023 05:33:07 +0900 Subject: [PATCH 13/31] Update Document --- README.md | 15 +++++ docs/strategies/circuit-breaker.md | 89 ++--------------------------- src/Snippets/Docs/CircuitBreaker.cs | 15 +++++ 3 files changed, 34 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 5eac80019b9..880ed3f857a 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,21 @@ new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOpti ShouldHandle = new PredicateBuilder().Handle() }); +// Adds a circuit breaker with dynamic break duration: +// +// Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. +// The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. +// The specified BreakDuration = TimeSpan.FromSeconds(30) will not be used due to the dynamic BreakDurationGenerator. +new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions +{ + FailureRatio = 0.5, + SamplingDuration = TimeSpan.FromSeconds(10), + MinimumThroughput = 8, + BreakDuration = TimeSpan.FromSeconds(30), + BreakDurationGenerator = static args => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)), + ShouldHandle = new PredicateBuilder().Handle(), +}); + // Handle specific failed results for HttpResponseMessage: new ResiliencePipelineBuilder() .AddCircuitBreaker(new CircuitBreakerStrategyOptions diff --git a/docs/strategies/circuit-breaker.md b/docs/strategies/circuit-breaker.md index e71e7012045..45db507859e 100644 --- a/docs/strategies/circuit-breaker.md +++ b/docs/strategies/circuit-breaker.md @@ -41,13 +41,15 @@ new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOpti // // Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. // The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. +// The specified BreakDuration = TimeSpan.FromSeconds(30) will not be used due to the dynamic BreakDurationGenerator. new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions { FailureRatio = 0.5, SamplingDuration = TimeSpan.FromSeconds(10), MinimumThroughput = 8, + BreakDuration = TimeSpan.FromSeconds(30), + BreakDurationGenerator = static args => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)), ShouldHandle = new PredicateBuilder().Handle(), - BreakDurationGenerator = (args) => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)) }); // Handle specific failed results for HttpResponseMessage: @@ -415,90 +417,7 @@ var retry = new ResiliencePipelineBuilder() - The Retry strategy fetches the sleep duration dynamically without knowing any specific knowledge about the Circuit Breaker. - If adjustments are needed for the `BreakDuration`, they can be made in one place. -### 2 - Using different duration for breaks - -In the case of Retry you can specify dynamically the sleep duration via the `DelayGenerator`. - -In the case of Circuit Breaker the `BreakDuration` is considered constant (can't be changed between breaks). - -❌ DON'T - -Use `Task.Delay` inside `OnOpened`: - - -```cs -static IEnumerable GetSleepDuration() -{ - for (int i = 1; i < 10; i++) - { - yield return TimeSpan.FromSeconds(i); - } -} - -var sleepDurationProvider = GetSleepDuration().GetEnumerator(); -sleepDurationProvider.MoveNext(); - -var circuitBreaker = new ResiliencePipelineBuilder() - .AddCircuitBreaker(new() - { - ShouldHandle = new PredicateBuilder().Handle(), - BreakDuration = TimeSpan.FromSeconds(0.5), - OnOpened = async args => - { - await Task.Delay(sleepDurationProvider.Current); - sleepDurationProvider.MoveNext(); - } - - }) - .Build(); -``` - - -**Reasoning**: - -- The minimum break duration value is half a second. This implies that each sleep lasts for `sleepDurationProvider.Current` plus an additional half a second. -- One might think that setting the `BreakDuration` to `sleepDurationProvider.Current` would address this, but it doesn't. This is because the `BreakDuration` is established only once and isn't re-assessed during each break. - - -```cs -circuitBreaker = new ResiliencePipelineBuilder() - .AddCircuitBreaker(new() - { - ShouldHandle = new PredicateBuilder().Handle(), - BreakDuration = sleepDurationProvider.Current, - OnOpened = async args => - { - Console.WriteLine($"Break: {sleepDurationProvider.Current}"); - sleepDurationProvider.MoveNext(); - } - - }) - .Build(); -``` - - -✅ DO - -Use the 'BreakDurationGenerator' to dynamically define the break duration: -```cs -// Adds a circuit breaker with dynamic break duration: -// -// Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. -// The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. -new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions -{ - FailureRatio = 0.5, - SamplingDuration = TimeSpan.FromSeconds(10), - MinimumThroughput = 8, - ShouldHandle = new PredicateBuilder().Handle(), - BreakDurationGenerator = (args) => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)) -}); -``` -**Reasoning**: - -- Using a BreakDurationGenerator like this, you can dynamically adjust the break duration with each failure. - -### 3 - Wrapping each endpoint with a circuit breaker +### 2 - Wrapping each endpoint with a circuit breaker Imagine that you have to call N number of services via `HttpClient`s. You want to decorate all downstream calls with the service-aware Circuit Breaker. diff --git a/src/Snippets/Docs/CircuitBreaker.cs b/src/Snippets/Docs/CircuitBreaker.cs index eba48b75dde..c3d345af733 100644 --- a/src/Snippets/Docs/CircuitBreaker.cs +++ b/src/Snippets/Docs/CircuitBreaker.cs @@ -28,6 +28,21 @@ public static async Task Usage() ShouldHandle = new PredicateBuilder().Handle() }); + // Adds a circuit breaker with dynamic break duration: + // + // Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. + // The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. + // The specified BreakDuration = TimeSpan.FromSeconds(30) will not be used due to the dynamic BreakDurationGenerator. + new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions + { + FailureRatio = 0.5, + SamplingDuration = TimeSpan.FromSeconds(10), + MinimumThroughput = 8, + BreakDuration = TimeSpan.FromSeconds(30), + BreakDurationGenerator = static args => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)), + ShouldHandle = new PredicateBuilder().Handle(), + }); + // Handle specific failed results for HttpResponseMessage: new ResiliencePipelineBuilder() .AddCircuitBreaker(new CircuitBreakerStrategyOptions From a1c84745cf78804b5ba8b01dc735d23a4d7913e2 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Sat, 28 Oct 2023 05:41:32 +0900 Subject: [PATCH 14/31] Update CircuitBreaker Document --- docs/strategies/circuit-breaker.md | 2 +- src/Snippets/Docs/CircuitBreaker.cs | 53 ++--------------------------- 2 files changed, 3 insertions(+), 52 deletions(-) diff --git a/docs/strategies/circuit-breaker.md b/docs/strategies/circuit-breaker.md index 45db507859e..958039d7363 100644 --- a/docs/strategies/circuit-breaker.md +++ b/docs/strategies/circuit-breaker.md @@ -426,7 +426,7 @@ You want to decorate all downstream calls with the service-aware Circuit Breaker Use a collection of Circuit Breakers and explicitly call `ExecuteAsync()`: - + ```cs // Defined in a common place var uriToCbMappings = new Dictionary diff --git a/src/Snippets/Docs/CircuitBreaker.cs b/src/Snippets/Docs/CircuitBreaker.cs index c3d345af733..c95be22202d 100644 --- a/src/Snippets/Docs/CircuitBreaker.cs +++ b/src/Snippets/Docs/CircuitBreaker.cs @@ -157,61 +157,12 @@ public static void Pattern_1() #endregion } - public static void AntiPattern_2() - { - #region circuit-breaker-anti-pattern-2 - static IEnumerable GetSleepDuration() - { - for (int i = 1; i < 10; i++) - { - yield return TimeSpan.FromSeconds(i); - } - } - - var sleepDurationProvider = GetSleepDuration().GetEnumerator(); - sleepDurationProvider.MoveNext(); - - var circuitBreaker = new ResiliencePipelineBuilder() - .AddCircuitBreaker(new() - { - ShouldHandle = new PredicateBuilder().Handle(), - BreakDuration = TimeSpan.FromSeconds(0.5), - OnOpened = async args => - { - await Task.Delay(sleepDurationProvider.Current); - sleepDurationProvider.MoveNext(); - } - - }) - .Build(); - - #endregion - - #region circuit-breaker-anti-pattern-2-ext - - circuitBreaker = new ResiliencePipelineBuilder() - .AddCircuitBreaker(new() - { - ShouldHandle = new PredicateBuilder().Handle(), - BreakDuration = sleepDurationProvider.Current, - OnOpened = async args => - { - Console.WriteLine($"Break: {sleepDurationProvider.Current}"); - sleepDurationProvider.MoveNext(); - } - - }) - .Build(); - - #endregion - } - - public static async ValueTask AntiPattern_3() + public static async ValueTask AntiPattern_2() { static ValueTask CallXYZOnDownstream1(CancellationToken ct) => ValueTask.CompletedTask; static ResiliencePipeline GetCircuitBreaker() => ResiliencePipeline.Empty; - #region circuit-breaker-anti-pattern-3 + #region circuit-breaker-anti-pattern-2 // Defined in a common place var uriToCbMappings = new Dictionary { From e24ab512ff18f15a3b94c5f77de0143d88e39988 Mon Sep 17 00:00:00 2001 From: LeeJonghoon <34878017+atawLee@users.noreply.github.com> Date: Sun, 29 Oct 2023 21:58:24 +0900 Subject: [PATCH 15/31] Apply suggestions from code review Update conventions and comments Co-authored-by: martintmk <103487740+martintmk@users.noreply.github.com> Co-authored-by: Martin Costello --- .../CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs | 4 ++-- .../CircuitBreaker/Controller/CircuitStateController.cs | 2 +- src/Snippets/Docs/CircuitBreaker.cs | 2 +- .../CircuitBreaker/Controller/CircuitStateControllerTests.cs | 3 +-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs index 7985e030628..c3b9c86d85c 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs @@ -70,10 +70,10 @@ public class CircuitBreakerStrategyOptions : ResilienceStrategyOptions public TimeSpan BreakDuration { get; set; } = CircuitBreakerConstants.DefaultBreakDuration; /// - /// Gets or sets an optional delegate to use to dynamically generate the break duration based on the health failure count. + /// Gets or sets an optional delegate to use to dynamically generate the break duration. /// /// - /// A delegate that takes an integer representing the health failure count and returns a indicating the break duration. + /// The default value is . /// public Func? BreakDurationGenerator { get; set; } diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs index 5eb834cf9f7..2de02031b4b 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs @@ -317,7 +317,7 @@ private void OpenCircuitFor_NeedsLock(Outcome outcome, TimeSpan breakDuration scheduledTask = null; var utcNow = _timeProvider.GetUtcNow(); - if (_breakDurationGenerator is not null && _behavior.FailureCount > 0) + if (_breakDurationGenerator is not null) { breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount)); } diff --git a/src/Snippets/Docs/CircuitBreaker.cs b/src/Snippets/Docs/CircuitBreaker.cs index c95be22202d..dd3609a6b66 100644 --- a/src/Snippets/Docs/CircuitBreaker.cs +++ b/src/Snippets/Docs/CircuitBreaker.cs @@ -32,7 +32,7 @@ public static async Task Usage() // // Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. // The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. - // The specified BreakDuration = TimeSpan.FromSeconds(30) will not be used due to the dynamic BreakDurationGenerator. + // The specified BreakDuration of 30 seconds will not be used due to the dynamic BreakDurationGenerator. new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions { FailureRatio = 0.5, diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs index 03cf654a24f..c7f28340244 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -511,6 +511,5 @@ private async Task OpenCircuit(CircuitStateController controller, Outcome Date: Sun, 29 Oct 2023 22:00:45 +0900 Subject: [PATCH 16/31] Apply suggestions from code review Update conventions and comments Co-authored-by: Martin Costello --- src/Snippets/Docs/CircuitBreaker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Snippets/Docs/CircuitBreaker.cs b/src/Snippets/Docs/CircuitBreaker.cs index dd3609a6b66..d9cf0a5f2ed 100644 --- a/src/Snippets/Docs/CircuitBreaker.cs +++ b/src/Snippets/Docs/CircuitBreaker.cs @@ -28,7 +28,7 @@ public static async Task Usage() ShouldHandle = new PredicateBuilder().Handle() }); - // Adds a circuit breaker with dynamic break duration: + // Adds a circuit breaker with a dynamic break duration: // // Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. // The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. From 92de0866af0c3516ba0b0f30ca43a416f85e33ae Mon Sep 17 00:00:00 2001 From: jonghoon Date: Tue, 31 Oct 2023 07:23:35 +0900 Subject: [PATCH 17/31] update document --- README.md | 4 +--- docs/strategies/circuit-breaker.md | 29 ++++++++++++++--------------- src/Snippets/Docs/CircuitBreaker.cs | 2 -- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index d463814714b..e4cb13180fc 100644 --- a/README.md +++ b/README.md @@ -229,11 +229,9 @@ var optionsComplex = new CircuitBreakerStrategyOptions ShouldHandle = new PredicateBuilder().Handle() }; -// Adds a circuit breaker with dynamic break duration: +// Adds a circuit breaker with a dynamic break duration: // // Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. -// The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. -// The specified BreakDuration = TimeSpan.FromSeconds(30) will not be used due to the dynamic BreakDurationGenerator. new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions { FailureRatio = 0.5, diff --git a/docs/strategies/circuit-breaker.md b/docs/strategies/circuit-breaker.md index c22d889732e..63ffb7665f3 100644 --- a/docs/strategies/circuit-breaker.md +++ b/docs/strategies/circuit-breaker.md @@ -36,11 +36,9 @@ var optionsComplex = new CircuitBreakerStrategyOptions ShouldHandle = new PredicateBuilder().Handle() }; -// Adds a circuit breaker with dynamic break duration: +// Adds a circuit breaker with a dynamic break duration: // // Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. -// The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. -// The specified BreakDuration = TimeSpan.FromSeconds(30) will not be used due to the dynamic BreakDurationGenerator. new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions { FailureRatio = 0.5, @@ -96,18 +94,19 @@ new ResiliencePipelineBuilder().AddCircuitBreaker(optionsSt ## Defaults -| Property | Default Value | Description | -| ------------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | -| `ShouldHandle` | Predicate that handles all exceptions except `OperationCanceledException`. | Specifies which results and exceptions are managed by the circuit breaker strategy. | -| `FailureRatio` | 0.1 | The ratio of failures to successes that will cause the circuit to break/open. | -| `MinimumThroughput` | 100 | The minimum number of actions that must occur in the circuit within a specific time slice. | -| `SamplingDuration` | 30 seconds | The time period over which failure ratios are calculated. | -| `BreakDuration` | 5 seconds | The time period for which the circuit will remain broken/open before attempting to reset. | -| `OnClosed` | `null` | Event triggered when the circuit transitions to the `Closed` state. | -| `OnOpened` | `null` | Event triggered when the circuit transitions to the `Opened` state. | -| `OnHalfOpened` | `null` | Event triggered when the circuit transitions to the `HalfOpened` state. | -| `ManualControl` | `null` | Allows for manual control to isolate or close the circuit. | -| `StateProvider` | `null` | Enables the retrieval of the current state of the circuit. | +| Property | Default Value | Description | +| ----------------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `ShouldHandle` | Predicate that handles all exceptions except `OperationCanceledException`. | Specifies which results and exceptions are managed by the circuit breaker strategy. | +| `FailureRatio` | 0.1 | The ratio of failures to successes that will cause the circuit to break/open. | +| `MinimumThroughput` | 100 | The minimum number of actions that must occur in the circuit within a specific time slice. | +| `SamplingDuration` | 30 seconds | The time period over which failure ratios are calculated. | +| `BreakDuration` | 5 seconds | The time period for which the circuit will remain broken/open before attempting to reset. | +| `BreakDurationGenerator`| `null` | A function to dynamically generate the break duration based on certain parameters. | +| `OnClosed` | `null` | Event triggered when the circuit transitions to the `Closed` state. | +| `OnOpened` | `null` | Event triggered when the circuit transitions to the `Opened` state. | +| `OnHalfOpened` | `null` | Event triggered when the circuit transitions to the `HalfOpened` state. | +| `ManualControl` | `null` | Allows for manual control to isolate or close the circuit. | +| `StateProvider` | `null` | Enables the retrieval of the current state of the circuit. | ## Diagrams diff --git a/src/Snippets/Docs/CircuitBreaker.cs b/src/Snippets/Docs/CircuitBreaker.cs index cdd2c431453..cb7ae22707a 100644 --- a/src/Snippets/Docs/CircuitBreaker.cs +++ b/src/Snippets/Docs/CircuitBreaker.cs @@ -30,8 +30,6 @@ public static async Task Usage() // Adds a circuit breaker with a dynamic break duration: // // Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. - // The duration is calculated as: minimum of (20 + 2^failureCount) seconds and capped at 400 seconds. - // The specified BreakDuration of 30 seconds will not be used due to the dynamic BreakDurationGenerator. new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions { FailureRatio = 0.5, From 8e5f8035f46db18b4d7316eacbf55bfbce9c61fc Mon Sep 17 00:00:00 2001 From: jonghoon Date: Tue, 31 Oct 2023 07:44:26 +0900 Subject: [PATCH 18/31] controller - Add context HealthInfo - Remove Default Parameter Argument - Missing XML documentation for the constructor and properties, ResilienceContext --- .../BreakDurationGeneratorArguments.cs | 19 ++++++++++++++----- .../Controller/CircuitStateController.cs | 2 +- .../CircuitBreaker/Health/HealthInfo.cs | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs index 1a23aa4ccde..69ba59b4546 100644 --- a/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs +++ b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs @@ -1,7 +1,3 @@ -using System.Collections.Generic; -using System; -using System.Runtime.CompilerServices; - namespace Polly.CircuitBreaker; /// @@ -11,13 +7,26 @@ public class BreakDurationGeneratorArguments { public BreakDurationGeneratorArguments( double failureRate, - int failureCount) + int failureCount, + ResilienceContext context) { FailureRate = failureRate; FailureCount = failureCount; + Context = context; } + /// + /// The failure rate that represents the ratio of failures to total actions. + /// public double FailureRate { get; set; } + /// + /// The count of failures that have occurred. + /// public int FailureCount { get; set; } + + /// + /// The context that provides additional information about the resilience operation. + /// + public ResilienceContext Context { get; set; } } diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs index 2de02031b4b..65368d12d55 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs @@ -319,7 +319,7 @@ private void OpenCircuitFor_NeedsLock(Outcome outcome, TimeSpan breakDuration if (_breakDurationGenerator is not null) { - breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount)); + breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount, context)); } _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; diff --git a/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs b/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs index 337cfb7c90d..e92e72d8f39 100644 --- a/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs +++ b/src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs @@ -1,6 +1,6 @@ namespace Polly.CircuitBreaker.Health; -internal readonly record struct HealthInfo(int Throughput, double FailureRate, int FailureCount = default(int)) +internal readonly record struct HealthInfo(int Throughput, double FailureRate, int FailureCount) { public static HealthInfo Create(int successes, int failures) { From 8a82bf07318a161a1f8c3b9fdce36b39e772f1da Mon Sep 17 00:00:00 2001 From: jonghoon Date: Tue, 31 Oct 2023 07:48:55 +0900 Subject: [PATCH 19/31] update unittest --- .../Controller/AdvancedCircuitBehaviorTests.cs | 13 +++++++------ .../Health/RollingHealthMetricsTests.cs | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs index def5bd3c73e..c7880770865 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs @@ -8,20 +8,21 @@ public class AdvancedCircuitBehaviorTests { private HealthMetrics _metrics = Substitute.For(TimeProvider.System); - [InlineData(10, 10, 0.0, 0.1, false)] - [InlineData(10, 10, 0.1, 0.1, true)] - [InlineData(10, 10, 0.2, 0.1, true)] - [InlineData(11, 10, 0.2, 0.1, true)] - [InlineData(9, 10, 0.1, 0.1, false)] + [InlineData(10, 10, 0.0, 0.1, 0, false)] + [InlineData(10, 10, 0.1, 0.1, 1, true)] + [InlineData(10, 10, 0.2, 0.1, 2, true)] + [InlineData(11, 10, 0.2, 0.1, 3, true)] + [InlineData(9, 10, 0.1, 0.1, 4, false)] [Theory] public void OnActionFailure_WhenClosed_EnsureCorrectBehavior( int throughput, int minimumThruput, double failureRate, double failureThreshold, + int failureCount, bool expectedShouldBreak) { - _metrics.GetHealthInfo().Returns(new HealthInfo(throughput, failureRate)); + _metrics.GetHealthInfo().Returns(new HealthInfo(throughput, failureRate, failureCount)); var behavior = new AdvancedCircuitBehavior(failureThreshold, minimumThruput, _metrics); diff --git a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs index 43a5549c6a2..29f372127a4 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs @@ -109,7 +109,7 @@ public void GetHealthInfo_SamplingDurationRespected(bool variance) _timeProvider.Advance(_samplingDuration + (variance ? TimeSpan.FromMilliseconds(1) : TimeSpan.Zero)); - metrics.GetHealthInfo().Should().Be(new HealthInfo(0, 0)); + metrics.GetHealthInfo().Should().Be(new HealthInfo(0, 0,0)); } private RollingHealthMetrics Create() => new(_samplingDuration, _windows, _timeProvider); From cf026599fca7163fd4f98256bd96cf5b142cdfe9 Mon Sep 17 00:00:00 2001 From: LeeJonghoon <34878017+atawLee@users.noreply.github.com> Date: Thu, 2 Nov 2023 21:01:27 +0900 Subject: [PATCH 20/31] Apply suggestions from code review Modify code conventions Co-authored-by: martintmk <103487740+martintmk@users.noreply.github.com> Co-authored-by: Martin Costello --- .../BreakDurationGeneratorArguments.cs | 14 +++++++------- .../Controller/AdvancedCircuitBehaviorTests.cs | 10 +++++----- .../Health/RollingHealthMetricsTests.cs | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs index 69ba59b4546..6d1625546c1 100644 --- a/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs +++ b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs @@ -3,7 +3,7 @@ namespace Polly.CircuitBreaker; /// /// Represents arguments used to generate a dynamic break duration for a circuit breaker. /// -public class BreakDurationGeneratorArguments +public readonly struct BreakDurationGeneratorArguments { public BreakDurationGeneratorArguments( double failureRate, @@ -16,17 +16,17 @@ public BreakDurationGeneratorArguments( } /// - /// The failure rate that represents the ratio of failures to total actions. + /// Gets the failure rate that represents the ratio of failures to total actions. /// - public double FailureRate { get; set; } + public double FailureRate { get; } /// - /// The count of failures that have occurred. + /// Gets the count of failures that have occurred. /// - public int FailureCount { get; set; } + public int FailureCount { get; } /// - /// The context that provides additional information about the resilience operation. + /// Gets the context that provides additional information about the resilience operation. /// - public ResilienceContext Context { get; set; } + public ResilienceContext Context { get; } } diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs index c7880770865..c256dda18a7 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs @@ -8,11 +8,11 @@ public class AdvancedCircuitBehaviorTests { private HealthMetrics _metrics = Substitute.For(TimeProvider.System); - [InlineData(10, 10, 0.0, 0.1, 0, false)] - [InlineData(10, 10, 0.1, 0.1, 1, true)] - [InlineData(10, 10, 0.2, 0.1, 2, true)] - [InlineData(11, 10, 0.2, 0.1, 3, true)] - [InlineData(9, 10, 0.1, 0.1, 4, false)] + [InlineData(10, 10, 0.0, 0.1, 0, false)] + [InlineData(10, 10, 0.1, 0.1, 1, true)] + [InlineData(10, 10, 0.2, 0.1, 2, true)] + [InlineData(11, 10, 0.2, 0.1, 3, true)] + [InlineData(9, 10, 0.1, 0.1, 4, false)] [Theory] public void OnActionFailure_WhenClosed_EnsureCorrectBehavior( int throughput, diff --git a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs index 29f372127a4..60efc1c3b09 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs @@ -109,7 +109,7 @@ public void GetHealthInfo_SamplingDurationRespected(bool variance) _timeProvider.Advance(_samplingDuration + (variance ? TimeSpan.FromMilliseconds(1) : TimeSpan.Zero)); - metrics.GetHealthInfo().Should().Be(new HealthInfo(0, 0,0)); + metrics.GetHealthInfo().Should().Be(new HealthInfo(0, 0, 0)); } private RollingHealthMetrics Create() => new(_samplingDuration, _windows, _timeProvider); From efcb36b0daba7fea360853f84a85d6d04eb9229e Mon Sep 17 00:00:00 2001 From: jonghoon Date: Thu, 2 Nov 2023 21:51:27 +0900 Subject: [PATCH 21/31] Reflect 'ValueTask' requirements --- .../CircuitBreakerStrategyOptions.TResult.cs | 2 +- .../CircuitBreaker/Controller/CircuitStateController.cs | 6 +++--- src/Snippets/Docs/CircuitBreaker.cs | 9 ++++++++- .../Controller/CircuitStateControllerTests.cs | 7 ++++--- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs index c3b9c86d85c..8d1bdbdeec1 100644 --- a/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs +++ b/src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs @@ -75,7 +75,7 @@ public class CircuitBreakerStrategyOptions : ResilienceStrategyOptions /// /// The default value is . /// - public Func? BreakDurationGenerator { get; set; } + public Func>? BreakDurationGenerator { get; set; } /// /// Gets or sets a predicate that determines whether the outcome should be handled by the circuit breaker. diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs index 65368d12d55..757da7f5cc0 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs @@ -16,7 +16,7 @@ internal sealed class CircuitStateController : IDisposable private readonly ResilienceStrategyTelemetry _telemetry; private readonly CircuitBehavior _behavior; private readonly TimeSpan _breakDuration; - private readonly Func? _breakDurationGenerator; + private readonly Func>? _breakDurationGenerator; private DateTimeOffset _blockedUntil; private CircuitState _circuitState = CircuitState.Closed; private Outcome? _lastOutcome; @@ -31,7 +31,7 @@ public CircuitStateController( CircuitBehavior behavior, TimeProvider timeProvider, ResilienceStrategyTelemetry telemetry, - Func? breakDurationGenerator = null) + Func>? breakDurationGenerator = null) { _breakDuration = breakDuration; _onOpened = onOpened; @@ -319,7 +319,7 @@ private void OpenCircuitFor_NeedsLock(Outcome outcome, TimeSpan breakDuration if (_breakDurationGenerator is not null) { - breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount, context)); + breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount, context)).GetAwaiter().GetResult(); } _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; diff --git a/src/Snippets/Docs/CircuitBreaker.cs b/src/Snippets/Docs/CircuitBreaker.cs index cb7ae22707a..d89f8a0867a 100644 --- a/src/Snippets/Docs/CircuitBreaker.cs +++ b/src/Snippets/Docs/CircuitBreaker.cs @@ -32,12 +32,19 @@ public static async Task Usage() // Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions { + Name = null, FailureRatio = 0.5, SamplingDuration = TimeSpan.FromSeconds(10), MinimumThroughput = 8, BreakDuration = TimeSpan.FromSeconds(30), - BreakDurationGenerator = static args => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)), + BreakDurationGenerator = static args => + ValueTask.FromResult(TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400))), ShouldHandle = new PredicateBuilder().Handle(), + OnClosed = null, + OnOpened = null, + OnHalfOpened = null, + ManualControl = null, + StateProvider = null, }); // Handle specific failed results for HttpResponseMessage: diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs index c7f28340244..a34ef0d8a17 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -307,14 +307,15 @@ public async Task OnActionFailureAsync_EnsureCorrectBehavior(CircuitState state, public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() { // arrange - var result = new TimeSpan(0, 0, 10, 0); + Func> result = () => new ValueTask(new TimeSpan(0, 0, 10, 0)); + using var controller = CreateController(new () { FailureRatio = 0, MinimumThroughput = 0, SamplingDuration = default, BreakDuration = TimeSpan.FromMinutes(1), - BreakDurationGenerator = (failureCount) => result, + BreakDurationGenerator = (failureCount) => result.Invoke(), OnClosed = null, OnOpened = null, OnHalfOpened = null, @@ -323,7 +324,7 @@ public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() }); await TransitionToState(controller, CircuitState.Closed); - var utcNow = DateTimeOffset.MaxValue - result; + var utcNow = DateTimeOffset.MaxValue - result().GetAwaiter().GetResult(); _timeProvider.SetUtcNow(utcNow); _circuitBehavior.FailureCount.Returns(1); From fc80c3e2598f3c50f8b13c580aab81931a284042 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Thu, 2 Nov 2023 23:15:40 +0900 Subject: [PATCH 22/31] Fixed warning --- .../CircuitBreaker/BreakDurationGeneratorArguments.cs | 11 ++++++++++- src/Polly.Core/PublicAPI.Unshipped.txt | 6 ++++++ .../Controller/CircuitStateControllerTests.cs | 10 +++++----- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs index 6d1625546c1..0b240a352c5 100644 --- a/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs +++ b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs @@ -1,10 +1,19 @@ namespace Polly.CircuitBreaker; - +#pragma warning disable CA1815 // Override equals and operator equals on value types /// /// Represents arguments used to generate a dynamic break duration for a circuit breaker. /// public readonly struct BreakDurationGeneratorArguments { + /// + /// Initializes a new instance of the struct. + /// + /// The failure rate at which the circuit breaker should trip. + /// It represents the ratio of failed actions to the total executed actions. + /// The number of failures that have occurred. + /// This count is used to determine if the failure threshold has been reached. + /// The resilience context providing additional information + /// about the execution state and failures. public BreakDurationGeneratorArguments( double failureRate, int failureCount, diff --git a/src/Polly.Core/PublicAPI.Unshipped.txt b/src/Polly.Core/PublicAPI.Unshipped.txt index ab058de62d4..9cb41d2d18c 100644 --- a/src/Polly.Core/PublicAPI.Unshipped.txt +++ b/src/Polly.Core/PublicAPI.Unshipped.txt @@ -1 +1,7 @@ #nullable enable +Polly.CircuitBreaker.BreakDurationGeneratorArguments +Polly.CircuitBreaker.BreakDurationGeneratorArguments.BreakDurationGeneratorArguments() -> void +Polly.CircuitBreaker.BreakDurationGeneratorArguments.BreakDurationGeneratorArguments(double failureRate, int failureCount, Polly.ResilienceContext! context) -> void +Polly.CircuitBreaker.BreakDurationGeneratorArguments.Context.get -> Polly.ResilienceContext! +Polly.CircuitBreaker.BreakDurationGeneratorArguments.FailureCount.get -> int +Polly.CircuitBreaker.BreakDurationGeneratorArguments.FailureRate.get -> double diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs index a34ef0d8a17..c6dbb5856b7 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -308,23 +308,24 @@ public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() { // arrange Func> result = () => new ValueTask(new TimeSpan(0, 0, 10, 0)); - - using var controller = CreateController(new () + using var controller = CreateController(new() { FailureRatio = 0, MinimumThroughput = 0, SamplingDuration = default, BreakDuration = TimeSpan.FromMinutes(1), - BreakDurationGenerator = (failureCount) => result.Invoke(), + BreakDurationGenerator = static args => new ValueTask(TimeSpan.FromMinutes(args.FailureCount)), OnClosed = null, OnOpened = null, OnHalfOpened = null, ManualControl = null, StateProvider = null }); - + await TransitionToState(controller, CircuitState.Closed); +#pragma warning disable CA2012 var utcNow = DateTimeOffset.MaxValue - result().GetAwaiter().GetResult(); +#pragma warning restore CA2012 _timeProvider.SetUtcNow(utcNow); _circuitBehavior.FailureCount.Returns(1); @@ -336,7 +337,6 @@ public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() // assert var blockedTill = GetBlockedTill(controller); - blockedTill.Should().Be(DateTimeOffset.MaxValue); } From 5b1a8825ea0ee4e041ab841b3682211bf4bd4a7f Mon Sep 17 00:00:00 2001 From: jonghoon Date: Thu, 2 Nov 2023 23:22:09 +0900 Subject: [PATCH 23/31] Fixed Warning --- .../Controller/CircuitStateController.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs index 757da7f5cc0..8e71fb79c7a 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs @@ -23,6 +23,7 @@ internal sealed class CircuitStateController : IDisposable private BrokenCircuitException _breakingException = new(); private bool _disposed; +#pragma warning disable S107 public CircuitStateController( TimeSpan breakDuration, Func, ValueTask>? onOpened, @@ -31,7 +32,8 @@ public CircuitStateController( CircuitBehavior behavior, TimeProvider timeProvider, ResilienceStrategyTelemetry telemetry, - Func>? breakDurationGenerator = null) + Func>? breakDurationGenerator = null) +#pragma warning restore S107 { _breakDuration = breakDuration; _onOpened = onOpened; @@ -319,11 +321,15 @@ private void OpenCircuitFor_NeedsLock(Outcome outcome, TimeSpan breakDuration if (_breakDurationGenerator is not null) { +#pragma warning disable CA2012 +#pragma warning disable S1226 breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount, context)).GetAwaiter().GetResult(); +#pragma warning restore S1226 +#pragma warning restore CA2012 } _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration; - + var transitionedState = _circuitState; _circuitState = CircuitState.Open; From 6962659c9b0808be808bc004e7a7724539877d2a Mon Sep 17 00:00:00 2001 From: jonghoon Date: Thu, 2 Nov 2023 23:23:12 +0900 Subject: [PATCH 24/31] update PublishAPI --- src/Polly.Core/PublicAPI.Unshipped.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Polly.Core/PublicAPI.Unshipped.txt b/src/Polly.Core/PublicAPI.Unshipped.txt index 9cb41d2d18c..a84ad565317 100644 --- a/src/Polly.Core/PublicAPI.Unshipped.txt +++ b/src/Polly.Core/PublicAPI.Unshipped.txt @@ -5,3 +5,5 @@ Polly.CircuitBreaker.BreakDurationGeneratorArguments.BreakDurationGeneratorArgum Polly.CircuitBreaker.BreakDurationGeneratorArguments.Context.get -> Polly.ResilienceContext! Polly.CircuitBreaker.BreakDurationGeneratorArguments.FailureCount.get -> int Polly.CircuitBreaker.BreakDurationGeneratorArguments.FailureRate.get -> double +Polly.CircuitBreaker.CircuitBreakerStrategyOptions.BreakDurationGenerator.get -> System.Func>? +Polly.CircuitBreaker.CircuitBreakerStrategyOptions.BreakDurationGenerator.set -> void From 5355b3e95a76a8ba50cace5514615b2f2a311e40 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Thu, 2 Nov 2023 23:42:31 +0900 Subject: [PATCH 25/31] Fixed Unittest --- .../Controller/CircuitStateControllerTests.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs index c6dbb5856b7..2605d1f57cc 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -307,7 +307,6 @@ public async Task OnActionFailureAsync_EnsureCorrectBehavior(CircuitState state, public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() { // arrange - Func> result = () => new ValueTask(new TimeSpan(0, 0, 10, 0)); using var controller = CreateController(new() { FailureRatio = 0, @@ -323,9 +322,9 @@ public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() }); await TransitionToState(controller, CircuitState.Closed); -#pragma warning disable CA2012 - var utcNow = DateTimeOffset.MaxValue - result().GetAwaiter().GetResult(); -#pragma warning restore CA2012 + + // utcNow DateTimeOffset.MaxValueԴϴ. , ִ밪̸, ⼭ ̻ ʽϴ. + var utcNow = DateTimeOffset.MaxValue; _timeProvider.SetUtcNow(utcNow); _circuitBehavior.FailureCount.Returns(1); @@ -337,7 +336,7 @@ public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() // assert var blockedTill = GetBlockedTill(controller); - blockedTill.Should().Be(DateTimeOffset.MaxValue); + blockedTill.Should().Be(utcNow); // ⼭ utcNow DateTimeOffset.MaxValue ϹǷ ̸ 밪 մϴ. } [InlineData(true)] From 5448c32edcbf1531601ea201d3c1701994b0089d Mon Sep 17 00:00:00 2001 From: jonghoon Date: Thu, 2 Nov 2023 23:44:48 +0900 Subject: [PATCH 26/31] Delete Korean Comment --- .../CircuitBreaker/Controller/CircuitStateControllerTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs index 2605d1f57cc..b047dea35e1 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs @@ -323,7 +323,6 @@ public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() await TransitionToState(controller, CircuitState.Closed); - // utcNow DateTimeOffset.MaxValueԴϴ. , ִ밪̸, ⼭ ̻ ʽϴ. var utcNow = DateTimeOffset.MaxValue; _timeProvider.SetUtcNow(utcNow); @@ -336,7 +335,7 @@ public async Task OnActionFailureAsync_EnsureBreakDurationGeneration() // assert var blockedTill = GetBlockedTill(controller); - blockedTill.Should().Be(utcNow); // ⼭ utcNow DateTimeOffset.MaxValue ϹǷ ̸ 밪 մϴ. + blockedTill.Should().Be(utcNow); } [InlineData(true)] From e4c3c7bb7ede038765a7cd85ae1bca83dcc12dc7 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Mon, 6 Nov 2023 18:08:02 +0900 Subject: [PATCH 27/31] Pull Main Branch --- .config/dotnet-tools.json | 2 +- .github/workflows/codeql-analysis.yml | 6 +++--- .github/workflows/ossf-scorecard.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 598c3707838..1bbff95cb81 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -15,7 +15,7 @@ ] }, "docfx": { - "version": "2.72.1", + "version": "2.73.0", "commands": [ "docfx" ] diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index f5f4f9fd886..ab989e9a7e0 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -44,14 +44,14 @@ jobs: restore-keys: ${{ runner.os }}-nuget- - name: Initialize CodeQL - uses: github/codeql-action/init@49abf0ba24d0b7953cb586944e918a0b92074c80 # v2.22.4 + uses: github/codeql-action/init@74483a38d39275f33fcff5f35b679b5ca4a26a99 # v2.22.5 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@49abf0ba24d0b7953cb586944e918a0b92074c80 # v2.22.4 + uses: github/codeql-action/autobuild@74483a38d39275f33fcff5f35b679b5ca4a26a99 # v2.22.5 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@49abf0ba24d0b7953cb586944e918a0b92074c80 # v2.22.4 + uses: github/codeql-action/analyze@74483a38d39275f33fcff5f35b679b5ca4a26a99 # v2.22.5 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index 3e5bd82a92c..cfbd8379e7a 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -39,6 +39,6 @@ jobs: retention-days: 5 - name: Upload to code-scanning - uses: github/codeql-action/upload-sarif@49abf0ba24d0b7953cb586944e918a0b92074c80 # v2.22.4 + uses: github/codeql-action/upload-sarif@74483a38d39275f33fcff5f35b679b5ca4a26a99 # v2.22.5 with: sarif_file: results.sarif From 7044ed448b8df81aac82fcbd06c0c36c9a59b0a7 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Mon, 6 Nov 2023 18:18:03 +0900 Subject: [PATCH 28/31] Merge Main Branch --- README.md | 13 -- docs/strategies/circuit-breaker.md | 196 +++++++++++++++++++++++----- src/Snippets/Docs/CircuitBreaker.cs | 121 ++++++++++++++--- 3 files changed, 265 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index e4cb13180fc..eb604743549 100644 --- a/README.md +++ b/README.md @@ -229,19 +229,6 @@ var optionsComplex = new CircuitBreakerStrategyOptions ShouldHandle = new PredicateBuilder().Handle() }; -// Adds a circuit breaker with a dynamic break duration: -// -// Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. -new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions -{ - FailureRatio = 0.5, - SamplingDuration = TimeSpan.FromSeconds(10), - MinimumThroughput = 8, - BreakDuration = TimeSpan.FromSeconds(30), - BreakDurationGenerator = static args => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)), - ShouldHandle = new PredicateBuilder().Handle(), -}); - // Handle specific failed results for HttpResponseMessage: var optionsShouldHandle = new CircuitBreakerStrategyOptions { diff --git a/docs/strategies/circuit-breaker.md b/docs/strategies/circuit-breaker.md index 63ffb7665f3..8353ba47b7a 100644 --- a/docs/strategies/circuit-breaker.md +++ b/docs/strategies/circuit-breaker.md @@ -36,19 +36,6 @@ var optionsComplex = new CircuitBreakerStrategyOptions ShouldHandle = new PredicateBuilder().Handle() }; -// Adds a circuit breaker with a dynamic break duration: -// -// Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. -new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions -{ - FailureRatio = 0.5, - SamplingDuration = TimeSpan.FromSeconds(10), - MinimumThroughput = 8, - BreakDuration = TimeSpan.FromSeconds(30), - BreakDurationGenerator = static args => TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400)), - ShouldHandle = new PredicateBuilder().Handle(), -}); - // Handle specific failed results for HttpResponseMessage: var optionsShouldHandle = new CircuitBreakerStrategyOptions { @@ -94,19 +81,18 @@ new ResiliencePipelineBuilder().AddCircuitBreaker(optionsSt ## Defaults -| Property | Default Value | Description | -| ----------------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -| `ShouldHandle` | Predicate that handles all exceptions except `OperationCanceledException`. | Specifies which results and exceptions are managed by the circuit breaker strategy. | -| `FailureRatio` | 0.1 | The ratio of failures to successes that will cause the circuit to break/open. | -| `MinimumThroughput` | 100 | The minimum number of actions that must occur in the circuit within a specific time slice. | -| `SamplingDuration` | 30 seconds | The time period over which failure ratios are calculated. | -| `BreakDuration` | 5 seconds | The time period for which the circuit will remain broken/open before attempting to reset. | -| `BreakDurationGenerator`| `null` | A function to dynamically generate the break duration based on certain parameters. | -| `OnClosed` | `null` | Event triggered when the circuit transitions to the `Closed` state. | -| `OnOpened` | `null` | Event triggered when the circuit transitions to the `Opened` state. | -| `OnHalfOpened` | `null` | Event triggered when the circuit transitions to the `HalfOpened` state. | -| `ManualControl` | `null` | Allows for manual control to isolate or close the circuit. | -| `StateProvider` | `null` | Enables the retrieval of the current state of the circuit. | +| Property | Default Value | Description | +| ------------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| `ShouldHandle` | Predicate that handles all exceptions except `OperationCanceledException`. | Specifies which results and exceptions are managed by the circuit breaker strategy. | +| `FailureRatio` | 0.1 | The ratio of failures to successes that will cause the circuit to break/open. | +| `MinimumThroughput` | 100 | The minimum number of actions that must occur in the circuit within a specific time slice. | +| `SamplingDuration` | 30 seconds | The time period over which failure ratios are calculated. | +| `BreakDuration` | 5 seconds | The time period for which the circuit will remain broken/open before attempting to reset. | +| `OnClosed` | `null` | Event triggered when the circuit transitions to the `Closed` state. | +| `OnOpened` | `null` | Event triggered when the circuit transitions to the `Opened` state. | +| `OnHalfOpened` | `null` | Event triggered when the circuit transitions to the `HalfOpened` state. | +| `ManualControl` | `null` | Allows for manual control to isolate or close the circuit. | +| `StateProvider` | `null` | Enables the retrieval of the current state of the circuit. | ## Diagrams @@ -322,7 +308,7 @@ sequenceDiagram Over the years, many developers have used Polly in various ways. Some of these recurring patterns may not be ideal. This section highlights the recommended practices and those to avoid. -### 1 - Using different sleep duration between retry attempts based on Circuit Breaker state +### Using different sleep duration between retry attempts based on Circuit Breaker state Imagine that we have an inner Circuit Breaker and an outer Retry strategies. @@ -332,7 +318,7 @@ We would like to define the retry in a way that the sleep duration calculation i Use a closure to branch based on circuit breaker state: - + ```cs var stateProvider = new CircuitBreakerStateProvider(); var circuitBreaker = new ResiliencePipelineBuilder() @@ -376,7 +362,7 @@ var retry = new ResiliencePipelineBuilder() Use `Context` to pass information between strategies: - + ```cs var circuitBreaker = new ResiliencePipelineBuilder() .AddCircuitBreaker(new() @@ -420,7 +406,73 @@ var retry = new ResiliencePipelineBuilder() - The Retry strategy fetches the sleep duration dynamically without knowing any specific knowledge about the Circuit Breaker. - If adjustments are needed for the `BreakDuration`, they can be made in one place. -### 2 - Wrapping each endpoint with a circuit breaker +### Using different duration for breaks + +In the case of Retry you can specify dynamically the sleep duration via the `DelayGenerator`. + +In the case of Circuit Breaker the `BreakDuration` is considered constant (can't be changed between breaks). + +❌ DON'T + +Use `Task.Delay` inside `OnOpened`: + + +```cs +static IEnumerable GetSleepDuration() +{ + for (int i = 1; i < 10; i++) + { + yield return TimeSpan.FromSeconds(i); + } +} + +var sleepDurationProvider = GetSleepDuration().GetEnumerator(); +sleepDurationProvider.MoveNext(); + +var circuitBreaker = new ResiliencePipelineBuilder() + .AddCircuitBreaker(new() + { + ShouldHandle = new PredicateBuilder().Handle(), + BreakDuration = TimeSpan.FromSeconds(0.5), + OnOpened = async args => + { + await Task.Delay(sleepDurationProvider.Current); + sleepDurationProvider.MoveNext(); + } + + }) + .Build(); +``` + + +**Reasoning**: + +- The minimum break duration value is half a second. This implies that each sleep lasts for `sleepDurationProvider.Current` plus an additional half a second. +- One might think that setting the `BreakDuration` to `sleepDurationProvider.Current` would address this, but it doesn't. This is because the `BreakDuration` is established only once and isn't re-assessed during each break. + + +```cs +circuitBreaker = new ResiliencePipelineBuilder() + .AddCircuitBreaker(new() + { + ShouldHandle = new PredicateBuilder().Handle(), + BreakDuration = sleepDurationProvider.Current, + OnOpened = async args => + { + Console.WriteLine($"Break: {sleepDurationProvider.Current}"); + sleepDurationProvider.MoveNext(); + } + + }) + .Build(); +``` + + +✅ DO + +The `CircuitBreakerStrategyOptions` currently do not support defining break durations dynamically. This may be re-evaluated in the future. For now, refer to the first example for a potential workaround. However, please use it with caution. + +### Wrapping each endpoint with a circuit breaker Imagine that you have to call N number of services via `HttpClient`s. You want to decorate all downstream calls with the service-aware Circuit Breaker. @@ -429,7 +481,7 @@ You want to decorate all downstream calls with the service-aware Circuit Breaker Use a collection of Circuit Breakers and explicitly call `ExecuteAsync()`: - + ```cs // Defined in a common place var uriToCbMappings = new Dictionary @@ -483,3 +535,85 @@ public Downstream1Client( > The above sample code used the `AsAsyncPolicy()` method to convert the `ResiliencePipeline` to `IAsyncPolicy`. > It is required because the `AddPolicyHandler()` method anticipates an `IAsyncPolicy` parameter. > Please be aware that, later an `AddResilienceHandler()` will be introduced in the `Microsoft.Extensions.Http.Resilience` package which is the successor of the `Microsoft.Extensions.Http.Polly`. + +### Reducing thrown exceptions + +In case of Circuit Breaker when it is either in the `Open` or `Isolated` state new requests are rejected immediately. + +That means the strategy will throw either a `BrokenCircuitException` or an `IsolatedCircuitException` respectively. + +❌ DON'T + +Use guard expression to call `Execute{Async}` only if the circuit is not broken: + + +```cs +var stateProvider = new CircuitBreakerStateProvider(); +var circuitBreaker = new ResiliencePipelineBuilder() + .AddCircuitBreaker(new() + { + ShouldHandle = new PredicateBuilder().Handle(), + BreakDuration = TimeSpan.FromSeconds(0.5), + StateProvider = stateProvider + }) + .Build(); + +if (stateProvider.CircuitState + is not CircuitState.Open + and not CircuitState.Isolated) +{ + var response = await circuitBreaker.ExecuteAsync(static async ct => + { + return await IssueRequest(); + }, CancellationToken.None); + + // Your code goes here to process response +} +``` + + +**Reasoning**: + +- The problem with this approach is that the circuit breaker will never transition into the `HalfOpen` state. +- The circuit breaker does not act as an active object. In other words the state transition does not happen automatically in the background. +- The circuit transition into the `HalfOpen` state when the `Execute{Async}` method is called and the `BreakDuration` elapsed. + +✅ DO + +Use `ExecuteOutcomeAsync` to avoid throwing exception: + + +```cs +var context = ResilienceContextPool.Shared.Get(); +var circuitBreaker = new ResiliencePipelineBuilder() + .AddCircuitBreaker(new() + { + ShouldHandle = new PredicateBuilder().Handle(), + BreakDuration = TimeSpan.FromSeconds(0.5), + }) + .Build(); + +Outcome outcome = await circuitBreaker.ExecuteOutcomeAsync(static async (ctx, state) => +{ + var response = await IssueRequest(); + return Outcome.FromResult(response); +}, context, "state"); + +ResilienceContextPool.Shared.Return(context); + +if (outcome.Exception is BrokenCircuitException) +{ + // The execution was stopped by the circuit breaker +} +else +{ + HttpResponseMessage response = outcome.Result!; + // Your code goes here to process the response +} +``` + + +**Reasoning**: + +- The `ExecuteOutcomeAsync` is a low-allocation API which does not throw exceptions; rather it captures them inside an `Outcome` data structure. +- Since you are calling one of the `Execute` methods, that's why the circuit breaker can transition into the `HalfOpen` state. diff --git a/src/Snippets/Docs/CircuitBreaker.cs b/src/Snippets/Docs/CircuitBreaker.cs index 66b49607db8..ec454febba2 100644 --- a/src/Snippets/Docs/CircuitBreaker.cs +++ b/src/Snippets/Docs/CircuitBreaker.cs @@ -27,26 +27,6 @@ public static async Task Usage() ShouldHandle = new PredicateBuilder().Handle() }; - // Adds a circuit breaker with a dynamic break duration: - // - // Same circuit breaking conditions as above, but with a dynamic break duration based on the failure count. - new ResiliencePipelineBuilder().AddCircuitBreaker(new CircuitBreakerStrategyOptions - { - Name = null, - FailureRatio = 0.5, - SamplingDuration = TimeSpan.FromSeconds(10), - MinimumThroughput = 8, - BreakDuration = TimeSpan.FromSeconds(30), - BreakDurationGenerator = static args => - ValueTask.FromResult(TimeSpan.FromSeconds(Math.Min(20 + Math.Pow(2, args.FailureCount), 400))), - ShouldHandle = new PredicateBuilder().Handle(), - OnClosed = null, - OnOpened = null, - OnHalfOpened = null, - ManualControl = null, - StateProvider = null, - }); - // Handle specific failed results for HttpResponseMessage: var optionsShouldHandle = new CircuitBreakerStrategyOptions { @@ -164,7 +144,106 @@ public static void Pattern_CircuitAwareRetry() .Build(); #endregion - + } + + public static void AntiPattern_SleepDurationGenerator() + { + #region circuit-breaker-anti-pattern-sleep-duration-generator + static IEnumerable GetSleepDuration() + { + for (int i = 1; i < 10; i++) + { + yield return TimeSpan.FromSeconds(i); + } + } + + var sleepDurationProvider = GetSleepDuration().GetEnumerator(); + sleepDurationProvider.MoveNext(); + + var circuitBreaker = new ResiliencePipelineBuilder() + .AddCircuitBreaker(new() + { + ShouldHandle = new PredicateBuilder().Handle(), + BreakDuration = TimeSpan.FromSeconds(0.5), + OnOpened = async args => + { + await Task.Delay(sleepDurationProvider.Current); + sleepDurationProvider.MoveNext(); + } + + }) + .Build(); + + #endregion + + #region circuit-breaker-anti-pattern-sleep-duration-generator-ext + + circuitBreaker = new ResiliencePipelineBuilder() + .AddCircuitBreaker(new() + { + ShouldHandle = new PredicateBuilder().Handle(), + BreakDuration = sleepDurationProvider.Current, + OnOpened = async args => + { + Console.WriteLine($"Break: {sleepDurationProvider.Current}"); + sleepDurationProvider.MoveNext(); + } + + }) + .Build(); + + #endregion + } + + public static async ValueTask AntiPattern_CircuitPerEndpoint() + { + static ValueTask CallXYZOnDownstream1(CancellationToken ct) => ValueTask.CompletedTask; + static ResiliencePipeline GetCircuitBreaker() => ResiliencePipeline.Empty; + + #region circuit-breaker-anti-pattern-cb-per-endpoint + // Defined in a common place + var uriToCbMappings = new Dictionary + { + [new Uri("https://downstream1.com")] = GetCircuitBreaker(), + // ... + [new Uri("https://downstreamN.com")] = GetCircuitBreaker() + }; + + // Used in the downstream 1 client + var downstream1Uri = new Uri("https://downstream1.com"); + await uriToCbMappings[downstream1Uri].ExecuteAsync(CallXYZOnDownstream1, CancellationToken.None); + #endregion + } + + private static ValueTask IssueRequest() => ValueTask.FromResult(new HttpResponseMessage()); + public static async ValueTask AntiPattern_ReduceThrownExceptions() + { + #region circuit-breaker-anti-pattern-reduce-thrown-exceptions + + var stateProvider = new CircuitBreakerStateProvider(); + var circuitBreaker = new ResiliencePipelineBuilder() + .AddCircuitBreaker(new() + { + ShouldHandle = new PredicateBuilder().Handle(), + BreakDuration = TimeSpan.FromSeconds(0.5), + StateProvider = stateProvider + }) + .Build(); + + if (stateProvider.CircuitState + is not CircuitState.Open + and not CircuitState.Isolated) + { + var response = await circuitBreaker.ExecuteAsync(static async ct => + { + return await IssueRequest(); + }, CancellationToken.None); + + // Your code goes here to process response + } + + #endregion + } public static async ValueTask Pattern_ReduceThrownExceptions() { From 886743615f1dc15891bd64f4d76386f25c62a344 Mon Sep 17 00:00:00 2001 From: jonghoon Date: Tue, 7 Nov 2023 22:41:00 +0900 Subject: [PATCH 29/31] Update Test Coverage --- .../BreakDurationGeneratorArguments.cs | 2 + .../Controller/CircuitStateController.cs | 2 - .../BreakDurationGeneratorArgumentsTests.cs | 41 +++++++++++++++++ .../AdvancedCircuitBehaviorTests.cs | 21 ++++++++- .../Health/HealthMetricsTests.cs | 46 +++++++++++++++++++ 5 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 test/Polly.Core.Tests/CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs diff --git a/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs index 0b240a352c5..6b10a0e8d9d 100644 --- a/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs +++ b/src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs @@ -1,5 +1,7 @@ namespace Polly.CircuitBreaker; + #pragma warning disable CA1815 // Override equals and operator equals on value types + /// /// Represents arguments used to generate a dynamic break duration for a circuit breaker. /// diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs index 8e71fb79c7a..2554648f42e 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs @@ -322,9 +322,7 @@ private void OpenCircuitFor_NeedsLock(Outcome outcome, TimeSpan breakDuration if (_breakDurationGenerator is not null) { #pragma warning disable CA2012 -#pragma warning disable S1226 breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount, context)).GetAwaiter().GetResult(); -#pragma warning restore S1226 #pragma warning restore CA2012 } diff --git a/test/Polly.Core.Tests/CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs b/test/Polly.Core.Tests/CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs new file mode 100644 index 00000000000..1746ef2376b --- /dev/null +++ b/test/Polly.Core.Tests/CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs @@ -0,0 +1,41 @@ +using Polly; +using Polly.CircuitBreaker; +namespace Polly.Core.Tests.CircuitBreaker; +public class BreakDurationGeneratorArgumentsTests +{ + [Fact] + public void Constructor_ShouldSetFailureRate() + { + double expectedFailureRate = 0.5; + int failureCount = 10; + var context = new ResilienceContext(); + + var args = new BreakDurationGeneratorArguments(expectedFailureRate, failureCount, context); + + args.FailureRate.Should().Be(expectedFailureRate); + } + + [Fact] + public void Constructor_ShouldSetFailureCount() + { + double failureRate = 0.5; + int expectedFailureCount = 10; + var context = new ResilienceContext(); + + var args = new BreakDurationGeneratorArguments(failureRate, expectedFailureCount, context); + + args.FailureCount.Should().Be(expectedFailureCount); + } + + [Fact] + public void Constructor_ShouldSetContext() + { + double failureRate = 0.5; + int failureCount = 10; + var expectedContext = new ResilienceContext(); + + var args = new BreakDurationGeneratorArguments(failureRate, failureCount, expectedContext); + + args.Context.Should().Be(expectedContext); + } +} diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs index c256dda18a7..7453a77f1d8 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs @@ -42,7 +42,6 @@ public void OnActionFailure_State_EnsureCorrectCalls(CircuitState state, bool sh _metrics = Substitute.For(TimeProvider.System); var sut = Create(); - sut.OnActionFailure(state, out var shouldBreak); shouldBreak.Should().BeFalse(); @@ -67,6 +66,26 @@ public void OnCircuitClosed_Ok() _metrics.Received(1).Reset(); } + [Theory] + [InlineData(10, 0.0, 0)] + [InlineData(10, 0.1, 1)] + [InlineData(10, 0.2, 2)] + [InlineData(11, 0.2, 3)] + [InlineData(9, 0.1, 4)] + public void BehaviorProperties_ShouldReflectHealthInfoValues( + int throughput, double failureRate, int failureCount) + { + var anyFailureThreshold = 10; + var anyMinimumThruput = 100; + + _metrics.GetHealthInfo().Returns(new HealthInfo(throughput, failureRate, failureCount)); + var behavior = new AdvancedCircuitBehavior(anyFailureThreshold, anyMinimumThruput, _metrics); + + behavior.FailureCount.Should().Be(failureCount, "because the FailureCount should match the HealthInfo"); + behavior.FailureRate.Should().Be(failureRate, "because the FailureRate should match the HealthInfo"); + } + + private AdvancedCircuitBehavior Create() { return new(CircuitBreakerConstants.DefaultFailureRatio, CircuitBreakerConstants.DefaultMinimumThroughput, _metrics); diff --git a/test/Polly.Core.Tests/CircuitBreaker/Health/HealthMetricsTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Health/HealthMetricsTests.cs index 71628fc2af1..3a53d02515b 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Health/HealthMetricsTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Health/HealthMetricsTests.cs @@ -17,4 +17,50 @@ public void Create_Ok(int samplingDurationMs, Type expectedType) .Should() .BeOfType(expectedType); } + + [Fact] + public void HealthInfo_WithZeroTotal_ShouldSetValuesCorrectly() + { + // Arrange & Act + var result = HealthInfo.Create(0, 0); + + // Assert + result.Throughput.Should().Be(0); + result.FailureRate.Should().Be(0); + result.FailureCount.Should().Be(0); + } + + [Fact] + public void HealthInfo_ParameterizedConstructor_ShouldSetProperties() + { + // Arrange + int expectedThroughput = 100; + double expectedFailureRate = 0.25; + int expectedFailureCount = 25; + + // Act + var result = new HealthInfo(expectedThroughput, expectedFailureRate, expectedFailureCount); + + // Assert + result.Throughput.Should().Be(expectedThroughput); + result.FailureRate.Should().Be(expectedFailureRate); + result.FailureCount.Should().Be(expectedFailureCount); + } + + [Fact] + public void HealthInfo_Constructor_ShouldSetValuesCorrectly() + { + // Arrange + int throughput = 10; + double failureRate = 0.2; + int failureCount = 2; + + // Act + var result = new HealthInfo(throughput, failureRate, failureCount); + + // Assert + result.Throughput.Should().Be(throughput); + result.FailureRate.Should().Be(failureRate); + result.FailureCount.Should().Be(failureCount); + } } From fb81ee6f6a17e285580b2dd78eabfbfa9855f1b8 Mon Sep 17 00:00:00 2001 From: LeeJonghoon <34878017+atawLee@users.noreply.github.com> Date: Wed, 8 Nov 2023 05:34:09 +0900 Subject: [PATCH 30/31] Apply suggestions from code review modify convention Co-authored-by: Martin Costello --- .../CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs | 2 ++ .../Controller/AdvancedCircuitBehaviorTests.cs | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/Polly.Core.Tests/CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs b/test/Polly.Core.Tests/CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs index 1746ef2376b..5afa53fe6ae 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs @@ -1,6 +1,8 @@ using Polly; using Polly.CircuitBreaker; + namespace Polly.Core.Tests.CircuitBreaker; + public class BreakDurationGeneratorArgumentsTests { [Fact] diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs index 7453a77f1d8..c0cc0462207 100644 --- a/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs +++ b/test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs @@ -73,7 +73,9 @@ public void OnCircuitClosed_Ok() [InlineData(11, 0.2, 3)] [InlineData(9, 0.1, 4)] public void BehaviorProperties_ShouldReflectHealthInfoValues( - int throughput, double failureRate, int failureCount) + int throughput, + double failureRate, + int failureCount) { var anyFailureThreshold = 10; var anyMinimumThruput = 100; @@ -85,7 +87,6 @@ public void BehaviorProperties_ShouldReflectHealthInfoValues( behavior.FailureRate.Should().Be(failureRate, "because the FailureRate should match the HealthInfo"); } - private AdvancedCircuitBehavior Create() { return new(CircuitBreakerConstants.DefaultFailureRatio, CircuitBreakerConstants.DefaultMinimumThroughput, _metrics); From 01b35fce648015cd910414641bea07dfe846e49f Mon Sep 17 00:00:00 2001 From: jonghoon Date: Wed, 8 Nov 2023 06:12:45 +0900 Subject: [PATCH 31/31] #pragma warning disable S1226 Upon further review, it has been confirmed that the warning in question is valid, and after checking the git Action, it has become apparent that this part is causing the issue. Therefore, I will reapply the disable pragma for this section. --- .../CircuitBreaker/Controller/CircuitStateController.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs index 2554648f42e..8e71fb79c7a 100644 --- a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs +++ b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs @@ -322,7 +322,9 @@ private void OpenCircuitFor_NeedsLock(Outcome outcome, TimeSpan breakDuration if (_breakDurationGenerator is not null) { #pragma warning disable CA2012 +#pragma warning disable S1226 breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount, context)).GetAwaiter().GetResult(); +#pragma warning restore S1226 #pragma warning restore CA2012 }