Skip to content

feat: Support per request max retries. #2269

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 22, 2022
Merged

feat: Support per request max retries. #2269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions Src/Support/Google.Apis.Core/Http/ConfigurableMessageHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ public class ConfigurableMessageHandler : DelegatingHandler
/// </summary>
public const string CredentialKey = "__CredentialKey";

/// <summary>
/// Key for request specific max retries.
/// </summary>
public const string MaxRetriesKey = "__MaxRetriesKey";

/// <summary>The current API version of this client library.</summary>
private static readonly string ApiVersion = Google.Apis.Util.Utilities.GetLibraryVersion();

Expand Down Expand Up @@ -396,7 +401,8 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
loggingRequestId = Interlocked.Increment(ref _loggingRequestId).ToString("X8");
}

int triesRemaining = NumTries;
int maxRetries = GetEffectiveMaxRetries(request);
int triesRemaining = maxRetries;
int redirectRemaining = NumRedirects;

Exception lastException = null;
Expand Down Expand Up @@ -501,8 +507,8 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
{
Request = request,
Exception = lastException,
TotalTries = NumTries,
CurrentFailedTry = NumTries - triesRemaining,
TotalTries = maxRetries,
CurrentFailedTry = maxRetries - triesRemaining,
CancellationToken = cancellationToken
}).ConfigureAwait(false);
}
Expand Down Expand Up @@ -561,8 +567,8 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
{
Request = request,
Response = response,
TotalTries = NumTries,
CurrentFailedTry = NumTries - triesRemaining,
TotalTries = maxRetries,
CurrentFailedTry = maxRetries - triesRemaining,
CancellationToken = cancellationToken
};

Expand Down Expand Up @@ -667,6 +673,10 @@ private IHttpExecuteInterceptor GetEffectiveCredential(HttpRequestMessage reques
(request.Properties.TryGetValue(CredentialKey, out var cred) && cred is IHttpExecuteInterceptor callCredential)
? callCredential : Credential;

private int GetEffectiveMaxRetries(HttpRequestMessage request) =>
(request.Properties.TryGetValue(MaxRetriesKey, out var maxRetries) && maxRetries is int perRequestMaxRetries)
? perRequestMaxRetries : NumTries;

/// <summary>
/// Handles redirect if the response's status code is redirect, redirects are turned on, and the header has
/// a location.
Expand Down
183 changes: 93 additions & 90 deletions Src/Support/Google.Apis.Tests/Apis/Http/ConfigurableMessageHandlerTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,9 +259,9 @@ public void SendAsync_ExecuteInterceptor_AbnormalResponse_UnsuccessfulResponseHa
{
var handler = new InterceptorMessageHandler();
handler.InjectedResponseMessage = new HttpResponseMessage()
{
StatusCode = HttpStatusCode.ServiceUnavailable
};
{
StatusCode = HttpStatusCode.ServiceUnavailable
};

var configurableHanlder = new ConfigurableMessageHandler(handler);
var interceptor = new InterceptorMessageHandler.Interceptor();
Expand All @@ -278,6 +278,33 @@ public void SendAsync_ExecuteInterceptor_AbnormalResponse_UnsuccessfulResponseHa
}
}

[Fact]
public void SendAsync_ExecuteInterceptor_AbnormalResponse_UnsuccessfulResponseHandler_PerRequestMaxRetries()
{
var handler = new InterceptorMessageHandler();
handler.InjectedResponseMessage = new HttpResponseMessage()
{
StatusCode = HttpStatusCode.ServiceUnavailable
};

Copy link
Collaborator

@jskeet jskeet Nov 22, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: remove one blank line?

Copy link
Contributor Author

@amanda-tarafa amanda-tarafa Nov 22, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

var configurableHandler = new ConfigurableMessageHandler(handler);
var interceptor = new InterceptorMessageHandler.Interceptor();
configurableHandler.AddExecuteInterceptor(interceptor);
configurableHandler.AddUnsuccessfulResponseHandler(new TrueUnsuccessfulResponseHandler());
// Let's have this request retry for a little longer than default.
int perRequestRetries = configurableHandler.NumTries + 2;

using (var client = new HttpClient(configurableHandler))
{
var request = new HttpRequestMessage(HttpMethod.Get, "https://test-execute-interceptor");
request.Properties.Add(ConfigurableMessageHandler.MaxRetriesKey, perRequestRetries);

HttpResponseMessage response = client.SendAsync(request).Result;
Assert.Equal(perRequestRetries, interceptor.Calls);
Assert.Equal(perRequestRetries, handler.Calls);
}
}

#endregion

#region Unsuccessful reponse handler
Expand Down Expand Up @@ -326,8 +353,11 @@ public Task<bool> HandleResponseAsync(HandleUnsuccessfulResponseArgs args)
}
}

/// <summary>Test helper for testing unsuccessful response handlers.</summary>
private void SubtestSendAsyncUnsuccessfulReponseHanlder(HttpStatusCode code)
[Theory]
[CombinatorialData]
public void SendAsyncUnsuccessfulReponseHanlder(
[CombinatorialValues(HttpStatusCode.OK, HttpStatusCode.BadGateway, HttpStatusCode.ServiceUnavailable)] HttpStatusCode code,
[CombinatorialValues(null, 5)] int? maxRetries)
{
var handler = new UnsuccessfulResponseMessageHandler { ResponseStatusCode = code };

Expand All @@ -338,6 +368,7 @@ private void SubtestSendAsyncUnsuccessfulReponseHanlder(HttpStatusCode code)
using (var client = new HttpClient(configurableHanlder))
{
var request = new HttpRequestMessage(HttpMethod.Get, "https://test-unsuccessful-handler");
int expectedMaxRetries = MaybeSetMaxRetries(maxRetries, configurableHanlder.NumTries, request);

HttpResponseMessage response = client.SendAsync(request).Result;
Assert.Equal(code, response.StatusCode);
Expand All @@ -346,8 +377,8 @@ private void SubtestSendAsyncUnsuccessfulReponseHanlder(HttpStatusCode code)
// handles service unavailable responses
if (code == HttpStatusCode.ServiceUnavailable)
{
Assert.Equal(configurableHanlder.NumTries, unsuccessfulHandler.Calls);
Assert.Equal(configurableHanlder.NumTries, handler.Calls);
Assert.Equal(expectedMaxRetries, unsuccessfulHandler.Calls);
Assert.Equal(expectedMaxRetries, handler.Calls);
}
else
{
Expand All @@ -358,33 +389,6 @@ private void SubtestSendAsyncUnsuccessfulReponseHanlder(HttpStatusCode code)
}
}

/// <summary>Tests that unsuccessful response handler isn't called when the response is successful.</summary>
[Fact]
public void SendAsync_UnsuccessfulReponseHanlder_SuccessfulReponse()
{
SubtestSendAsyncUnsuccessfulReponseHanlder(HttpStatusCode.OK);
}

/// <summary>
/// Tests that unsuccessful response handler is called when the response is unsuccessful, but the handler can't
/// handle the abnormal response (e.g. different status code).
/// </summary>
[Fact]
public void SendAsync_UnsuccessfulReponseHanlder_AbnormalResponse_DifferentStatusCode()
{
SubtestSendAsyncUnsuccessfulReponseHanlder(HttpStatusCode.BadGateway);
}

/// <summary>
/// Tests that unsuccessful response handler is called when the response is unsuccessful and the handler can
/// handle the abnormal response (e.g. same status code).
/// </summary>
[Fact]
public void SendAsync_UnsuccessfulReponseHanlder_AbnormalResponse_SameStatusCode()
{
SubtestSendAsyncUnsuccessfulReponseHanlder(HttpStatusCode.ServiceUnavailable);
}

/// <summary>Tests abnormal response when unsuccessful response handler isn't plugged.</summary>
[Fact]
public void SendAsync_AbnormalResponse_WithoutUnsuccessfulReponseHandler()
Expand Down Expand Up @@ -464,8 +468,10 @@ public Task<bool> HandleExceptionAsync(HandleExceptionArgs args)
}
}

/// <summary>Subtest for exception handler which tests that exception handler is invoked.</summary>
private void SubtestSendAsyncExceptionHandler(bool throwException, bool handle)
[Theory]
[CombinatorialData]
public void SendAsyncExceptionHandler(bool throwException, bool handle,
[CombinatorialValues(null, 5)] int? maxRetries)
{
var handler = new ExceptionMessageHandler { ThrowException = throwException };

Expand All @@ -476,6 +482,8 @@ private void SubtestSendAsyncExceptionHandler(bool throwException, bool handle)
using (var client = new HttpClient(configurableHanlder))
{
var request = new HttpRequestMessage(HttpMethod.Get, "https://test-exception-handler");
int expectedMaxRetries = MaybeSetMaxRetries(maxRetries, configurableHanlder.NumTries, request);

try
{
HttpResponseMessage response = client.SendAsync(request).Result;
Expand All @@ -493,46 +501,18 @@ private void SubtestSendAsyncExceptionHandler(bool throwException, bool handle)
// only 1
if (throwException)
{
Assert.Equal(handle ? configurableHanlder.NumTries : 1, exceptionHandler.Calls);
Assert.Equal(handle ? expectedMaxRetries : 1, exceptionHandler.Calls);
}
// exception wasn't supposed to be thrown, so no call to exception handler should be made
else
{
Assert.Equal(0, exceptionHandler.Calls);
}

Assert.Equal(throwException & handle ? configurableHanlder.NumTries : 1, handler.Calls);
Assert.Equal(throwException & handle ? expectedMaxRetries : 1, handler.Calls);
}
}


/// <summary>Tests that the exception handler isn't called on successful response.</summary>
[Fact]
public void SendAsync_ExceptionHandler_SuccessReponse()
{
SubtestSendAsyncExceptionHandler(false, true);
}

/// <summary>
/// Tests that the exception handler is called when exception is thrown on execute, but it can't handle the
/// exception.
/// </summary>
[Fact]
public void SendAsync_ExceptionHandler_ThrowException_DontHandle()
{
SubtestSendAsyncExceptionHandler(true, false);
}

/// <summary>
/// Tests that the exception handler is called when exception is thrown on execute, and it handles the
/// exception.
/// </summary>
[Fact]
public void SendAsync_ExceptionHandler_ThrowException_Handle()
{
SubtestSendAsyncExceptionHandler(true, true);
}

/// <summary>Tests an exception is thrown on execute and there is no exception handler.</summary>
[Fact]
public void SendAsync_ThrowException_WithoutExceptionHandler()
Expand Down Expand Up @@ -571,78 +551,88 @@ public void SendAsync_ThrowException_WithoutExceptionHandler()
/// Tests that back-off handler works as expected when exception is thrown.
/// Use default max time span (2 minutes).
/// </summary>
[Fact]
public void SendAsync_BackOffExceptionHandler_Throw_Max2Minutes()
[Theory]
[InlineData(null)]
[InlineData(5)]
public void SendAsync_BackOffExceptionHandler_Throw_Max2Minutes(int? maxRetries)
{
// create exponential back-off without delta interval, so expected seconds are exactly 1, 2, 4, 8, etc.
var initializer = new BackOffHandler.Initializer(new ExponentialBackOff(TimeSpan.Zero));
SubtestSendAsync_BackOffExceptionHandler(true, initializer);
SubtestSendAsync_BackOffExceptionHandler(true, initializer, maxRetries: maxRetries);
}

/// <summary>
/// Tests that back-off handler works as expected when exception is thrown.
/// Max time span is set to 200 milliseconds (as a result the back-off handler can't handle the exception).
/// </summary>
[Fact]
public void SendAsync_BackOffExceptionHandler_Throw_Max200Milliseconds()
[Theory]
[InlineData(null)]
[InlineData(5)]
public void SendAsync_BackOffExceptionHandler_Throw_Max200Milliseconds(int? maxRetries)
{
var initializer = new BackOffHandler.Initializer(new ExponentialBackOff(TimeSpan.Zero))
{
MaxTimeSpan = TimeSpan.FromMilliseconds(200)
};
SubtestSendAsync_BackOffExceptionHandler(true, initializer);
SubtestSendAsync_BackOffExceptionHandler(true, initializer, maxRetries: maxRetries);
}

/// <summary>
/// Tests that back-off handler works as expected when exception is thrown.
/// Max time span is set to 1 hour.
/// </summary>
[Fact]
public void SendAsync_BackOffExceptionHandler_Throw_Max1Hour()
[Theory]
[InlineData(null)]
[InlineData(5)]
public void SendAsync_BackOffExceptionHandler_Throw_Max1Hour(int? maxRetries)
{
var initializer = new BackOffHandler.Initializer(new ExponentialBackOff(TimeSpan.Zero))
{
MaxTimeSpan = TimeSpan.FromHours(1)
};
SubtestSendAsync_BackOffExceptionHandler(true, initializer);
SubtestSendAsync_BackOffExceptionHandler(true, initializer,maxRetries: maxRetries);
}

/// <summary>
/// Tests that back-off handler works as expected when
/// <seealso cref="System.Threading.Tasks.TaskCanceledException"/>> is thrown.
/// </summary>
[Fact]
public void SendAsync_BackOffExceptionHandler_ThrowCanceledException()
[Theory]
[InlineData(null)]
[InlineData(5)]
public void SendAsync_BackOffExceptionHandler_ThrowCanceledException(int? maxRetries)
{
var initializer = new BackOffHandler.Initializer(new ExponentialBackOff(TimeSpan.Zero));
SubtestSendAsync_BackOffExceptionHandler(true, initializer, new TaskCanceledException());
SubtestSendAsync_BackOffExceptionHandler(true, initializer, new TaskCanceledException(), maxRetries);
}

/// <summary>
/// Tests that back-off handler works as expected with the not defaulted exception handler.
/// </summary>
[Fact]
public void SendAsync_BackOffExceptionHandler_DifferentHandler()
[Theory]
[InlineData(null)]
[InlineData(5)]
public void SendAsync_BackOffExceptionHandler_DifferentHandler(int? maxRetries)
{
var initializer = new BackOffHandler.Initializer(new ExponentialBackOff(TimeSpan.Zero));
initializer.HandleExceptionFunc = e => (e is InvalidCastException);
SubtestSendAsync_BackOffExceptionHandler(true, initializer, new InvalidCastException());
SubtestSendAsync_BackOffExceptionHandler(true, initializer, new InvalidCastException(), maxRetries);

initializer.HandleExceptionFunc = e => !(e is InvalidCastException);
SubtestSendAsync_BackOffExceptionHandler(true, initializer, new InvalidCastException());
SubtestSendAsync_BackOffExceptionHandler(true, initializer, new InvalidCastException(), maxRetries);
}

/// <summary>Tests that back-off handler works as expected when exception isn't thrown.</summary>
[Fact]
public void SendAsync_BackOffExceptionHandler_DontThrow()
[Theory]
[InlineData(null)]
[InlineData(5)]
public void SendAsync_BackOffExceptionHandler_DontThrow(int? maxRetries)
{
var initializer = new BackOffHandler.Initializer(new ExponentialBackOff(TimeSpan.Zero));
SubtestSendAsync_BackOffExceptionHandler(false, initializer);
SubtestSendAsync_BackOffExceptionHandler(false, initializer, maxRetries: maxRetries);
}

/// <summary>Subtest that back-off handler works as expected when exception is or isn't thrown.</summary>
private void SubtestSendAsync_BackOffExceptionHandler(bool throwException,
BackOffHandler.Initializer initializer, Exception exceptionToThrow = null)
private void SubtestSendAsync_BackOffExceptionHandler(bool throwException, BackOffHandler.Initializer initializer, Exception exceptionToThrow = null, int? maxRetries = null)
{
var handler = new ExceptionMessageHandler { ThrowException = throwException };
if (exceptionToThrow != null)
Expand All @@ -654,19 +644,21 @@ private void SubtestSendAsync_BackOffExceptionHandler(bool throwException,
var boHandler = new MockBackOffHandler(initializer);
configurableHanlder.AddExceptionHandler(boHandler);

var request = new HttpRequestMessage(HttpMethod.Get, "https://test-exception-handler");
int expectedMaxRetries = MaybeSetMaxRetries(maxRetries, configurableHanlder.NumTries, request);

int boHandleCount = 0;
// if an exception should be thrown and the handler can handle it then calculate the handle count by the
// lg(MaxTimeSpan)
if (throwException && initializer.HandleExceptionFunc(exceptionToThrow))
{
boHandleCount = Math.Min((int)Math.Floor(Math.Log(boHandler.MaxTimeSpan.TotalSeconds, 2)) + 1,
configurableHanlder.NumTries - 1);
expectedMaxRetries - 1);
boHandleCount = boHandleCount >= 0 ? boHandleCount : 0;
}

using (var client = new HttpClient(configurableHanlder))
{
var request = new HttpRequestMessage(HttpMethod.Get, "https://test-exception-handler");
try
{
HttpResponseMessage response = client.SendAsync(request).Result;
Expand Down Expand Up @@ -1216,5 +1208,16 @@ protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage reques
return Task.FromResult(new HttpResponseMessage());
}
}

private int MaybeSetMaxRetries(int? perRequestMaxRetries, int defaultMaxRetries, HttpRequestMessage requestMessage)
{
if (perRequestMaxRetries == null)
{
return defaultMaxRetries;
}
int configuredRetries = perRequestMaxRetries.Value;
requestMessage.Properties.Add(ConfigurableMessageHandler.MaxRetriesKey, configuredRetries);
return configuredRetries;
}
}
}