private static async Task <T> ThrowsException <T>(ITestServer server) where T : Exception
{
return(await server.ExecuteAsync(async address =>
{
// Arrange
var retryHandler = new HttpRetryHandler();
var countingHandler = new CountingHandler {
InnerHandler = new HttpClientHandler()
};
var httpClient = new HttpClient(countingHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, address))
{
MaxTries = 2,
RetryDelay = TimeSpan.Zero
};
// Act
Func <Task> actionAsync = () => retryHandler.SendAsync(
request,
new TestLogger(),
CancellationToken.None);
// Act & Assert
var exception = await Assert.ThrowsAsync <T>(actionAsync);
Assert.Equal(2, countingHandler.Hits);
return exception;
}));
}
public async Task HttpRetryHandler_DifferentRequestInstanceEachTime()
{
// Arrange
var requests = new HashSet <HttpRequestMessage>();
Func <HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
{
requests.Add(requestMessage);
return(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable));
};
var retryHandler = new HttpRetryHandler();
var testHandler = new TestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = MaxTries,
RequestTimeout = Timeout.InfiniteTimeSpan,
RetryDelay = TimeSpan.Zero
};
var log = new TestLogger();
// Act
using (await retryHandler.SendAsync(request, log, CancellationToken.None))
{
}
// Assert
Assert.Equal(MaxTries, requests.Count);
}
public async Task HttpRetryHandler_AddHeaders()
{
// Arrange
var retryHandler = new HttpRetryHandler();
var testHandler = new TestHandler();
using (var httpClient = new HttpClient(testHandler))
{
var request = new HttpRetryHandlerRequest(
httpClient,
() => new HttpRequestMessage(HttpMethod.Get, TestUrl));
var id = Guid.NewGuid().ToString();
request.AddHeaders.Add(new KeyValuePair <string, IEnumerable <string> >(ProtocolConstants.SessionId, new[] { id }));
var log = new TestLogger();
// Act
using (var actualResponse = await retryHandler.SendAsync(request, log, CancellationToken.None))
{
// Assert
testHandler.LastRequest.Headers.GetValues(ProtocolConstants.SessionId)
.FirstOrDefault()
.Should()
.Be(id);
}
}
}
public async Task HttpRetryHandler_404VerifySingleHit()
{
// Arrange
var hits = 0;
Func <HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
{
hits++;
return(new HttpResponseMessage(HttpStatusCode.OK));
};
var retryHandler = new HttpRetryHandler();
var testHandler = new HttpRetryTestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = MaxTries,
RequestTimeout = Timeout.InfiniteTimeSpan,
RetryDelay = TimeSpan.Zero
};
var log = new TestLogger();
// Act
using (var response = await retryHandler.SendAsync(request, log, CancellationToken.None))
{
// Assert
Assert.Equal(1, hits);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}
}
public async Task HttpRetryHandler_MultipleTriesNoSuccess()
{
// Arrange
var hits = 0;
Func <HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
{
hits++;
return(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable));
};
var retryHandler = new HttpRetryHandler();
var testHandler = new HttpRetryTestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = MaxTries,
RequestTimeout = Timeout.InfiniteTimeSpan,
RetryDelay = TimeSpan.Zero
};
var log = new TestLogger();
// Act
using (await retryHandler.SendAsync(request, log, CancellationToken.None))
{
}
// Assert
Assert.Equal(MaxTries, hits);
}
public async Task HttpRetryHandler_CancelsRequestAfterTimeout()
{
// Arrange
CancellationToken requestToken = CancellationToken.None;
Func <HttpRequestMessage, CancellationToken, Task <HttpResponseMessage> > handler = async(requestMessage, token) =>
{
requestToken = token;
await Task.Delay(LargeTimeout, token);
return(new HttpResponseMessage(HttpStatusCode.OK));
};
var retryHandler = new HttpRetryHandler();
var testHandler = new TestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = 1,
RequestTimeout = SmallTimeout,
RetryDelay = TimeSpan.Zero
};
// Act
Func <Task> actionAsync = () => retryHandler.SendAsync(
request,
new TestLogger(),
CancellationToken.None);
// Assert
await Assert.ThrowsAsync <TimeoutException>(actionAsync);
Assert.True(requestToken.CanBeCanceled);
Assert.True(requestToken.IsCancellationRequested);
}
public async Task HttpRetryHandler_EnhancedRetryAllowsSettingMoreRetries()
{
// Arrange
var tries = 0;
var sent503 = false;
Func <HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
{
tries++;
// Return 503 for the first 2 tries
if (tries > 10)
{
return(new HttpResponseMessage(HttpStatusCode.OK));
}
else
{
sent503 = true;
return(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable));
}
};
TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary <string, string>()
{
[EnhancedHttpRetryHelper.IsEnabledEnvironmentVariableName] = bool.TrueString,
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = "11",
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = "3"
});
EnhancedHttpRetryHelper helper = new EnhancedHttpRetryHelper(testEnvironmentVariableReader);
Assert.Equal(helper.IsEnabled, true);
// Enhanced retry mode causes a random 0-199 ms jitter so we can't time it in this test
// but we can make sure the setting got through
Assert.Equal(helper.DelayInMilliseconds, 3);
Assert.Equal(helper.RetryCount, 11);
var retryHandler = new HttpRetryHandler(testEnvironmentVariableReader);
var testHandler = new HttpRetryTestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = helper.RetryCount,
RequestTimeout = Timeout.InfiniteTimeSpan,
// HttpRetryHandler will override with values from NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS
// so set this to a value that will cause test timeout if the correct value is not honored.
RetryDelay = TimeSpan.FromMilliseconds(int.MaxValue) // = about 24 days
};
var log = new TestLogger();
// Act
using (var response = await retryHandler.SendAsync(request, log, CancellationToken.None))
{
// Assert
Assert.True(sent503);
Assert.Equal(11, tries);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}
}
public async Task HttpRetryHandler_MultipleTriesUntilSuccess()
{
// Arrange
TimeSpan retryDelay = TimeSpan.Zero;
TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary <string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = MaxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
var tries = 0;
var sent503 = false;
Func <HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
{
tries++;
// Return 503 for the first 2 tries
if (tries > 2)
{
return(new HttpResponseMessage(HttpStatusCode.OK));
}
else
{
sent503 = true;
return(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable));
}
};
var retryHandler = new HttpRetryHandler(testEnvironmentVariableReader);
var testHandler = new HttpRetryTestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = MaxTries,
RequestTimeout = Timeout.InfiniteTimeSpan,
RetryDelay = retryDelay
};
var log = new TestLogger();
// Act
using (var response = await retryHandler.SendAsync(request, log, CancellationToken.None))
{
// Assert
Assert.True(sent503);
Assert.Equal(3, tries);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}
}
public async Task HttpRetryHandler_AppliesTimeoutToRequestsIndividually()
{
// Arrange
// 20 requests that take 250ms each for a total of 5 seconds (plus noise).
var requestDuration = TimeSpan.FromMilliseconds(250);
var maxTries = 20;
var retryDelay = TimeSpan.Zero;
TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary <string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = maxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
// Make the request timeout longer than each request duration but less than the total
// duration of all attempts.
var requestTimeout = TimeSpan.FromMilliseconds(4000);
var hits = 0;
Func <HttpRequestMessage, CancellationToken, Task <HttpResponseMessage> > handler = async(requestMessage, token) =>
{
hits++;
await Task.Delay(requestDuration);
return(new HttpResponseMessage(HttpStatusCode.InternalServerError));
};
var retryHandler = new HttpRetryHandler(testEnvironmentVariableReader);
var testHandler = new HttpRetryTestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = maxTries,
RequestTimeout = requestTimeout,
RetryDelay = retryDelay
};
var log = new TestLogger();
// Act
using (await retryHandler.SendAsync(request, log, CancellationToken.None))
{
}
// Assert
Assert.Equal(maxTries, hits);
}
public async Task HttpRetryHandler_MultipleTriesTimed()
{
// Arrange
TimeSpan retryDelay = SmallTimeout;
TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary <string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = MaxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
Func <HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
{
return(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable));
};
var minTime = GetRetryMinTime(MaxTries, SmallTimeout);
var retryHandler = new HttpRetryHandler(testEnvironmentVariableReader);
var testHandler = new HttpRetryTestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = MaxTries,
RequestTimeout = Timeout.InfiniteTimeSpan,
RetryDelay = retryDelay
};
var log = new TestLogger();
// Act
var timer = new Stopwatch();
timer.Start();
using (await retryHandler.SendAsync(request, log, CancellationToken.None))
{
}
timer.Stop();
// Assert
Assert.True(
timer.Elapsed >= minTime,
$"Expected this to take at least: {minTime} But it finished in: {timer.Elapsed}");
}
public async Task HttpRetryHandler_TimesOutDownload()
{
// Arrange
var hits = 0;
var memoryStream = new MemoryStream(Encoding.ASCII.GetBytes("foobar"));
var expectedMilliseconds = 50;
Func <HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
{
hits++;
return(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(new SlowStream(memoryStream)
{
DelayPerByte = TimeSpan.FromSeconds(1)
})
});
};
var retryHandler = new HttpRetryHandler();
var testHandler = new TestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
DownloadTimeout = TimeSpan.FromMilliseconds(expectedMilliseconds)
};
var destinationStream = new MemoryStream();
var log = new TestLogger();
// Act
using (var response = await retryHandler.SendAsync(request, log, CancellationToken.None))
using (var stream = await response.Content.ReadAsStreamAsync())
{
var actual = await Assert.ThrowsAsync <IOException>(() => stream.CopyToAsync(destinationStream));
// Assert
Assert.Equal(1, hits);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.IsType <TimeoutException>(actual.InnerException);
Assert.EndsWith(
$"timed out because no data was received for {expectedMilliseconds}ms.",
actual.Message);
}
}
public async Task HttpRetryHandler_MultipleTriesUntilSuccess()
{
// Arrange
var tries = 0;
var sent503 = false;
Func <HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
{
tries++;
// Return 503 for the first 2 tries
if (tries > 2)
{
return(new HttpResponseMessage(HttpStatusCode.OK));
}
else
{
sent503 = true;
return(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable));
}
};
var retryHandler = new HttpRetryHandler();
var testHandler = new TestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = MaxTries,
RequestTimeout = Timeout.InfiniteTimeSpan,
RetryDelay = TimeSpan.Zero
};
var log = new TestLogger();
// Act
using (var response = await retryHandler.SendAsync(request, log, CancellationToken.None))
{
// Assert
Assert.True(sent503);
Assert.Equal(3, tries);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}
}
public async Task HttpRetryHandler_CancelsRequestAfterTimeout()
{
// Arrange
TimeSpan retryDelay = TimeSpan.Zero;
TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary <string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = MaxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
var requestToken = CancellationToken.None;
Func <HttpRequestMessage, CancellationToken, Task <HttpResponseMessage> > handler = async(requestMessage, token) =>
{
requestToken = token;
await Task.Delay(LargeTimeout, token);
return(new HttpResponseMessage(HttpStatusCode.OK));
};
var retryHandler = new HttpRetryHandler(testEnvironmentVariableReader);
var testHandler = new HttpRetryTestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = 1,
RequestTimeout = SmallTimeout,
RetryDelay = retryDelay
};
// Act
Func <Task> actionAsync = () => retryHandler.SendAsync(
request,
new TestLogger(),
CancellationToken.None);
// Assert
await Assert.ThrowsAsync <TimeoutException>(actionAsync);
Assert.True(requestToken.CanBeCanceled);
Assert.True(requestToken.IsCancellationRequested);
}
public async Task HttpRetryHandler_ReturnsContentHeaders()
{
// Arrange
var retryHandler = new HttpRetryHandler();
using (var httpClient = new HttpClient())
{
var request = new HttpRetryHandlerRequest(
httpClient,
() => new HttpRequestMessage(HttpMethod.Get, "https://api.nuget.org/v3/index.json"));
var log = new TestLogger();
// Act
using (var actualResponse = await retryHandler.SendAsync(request, log, CancellationToken.None))
{
// Assert
Assert.NotEmpty(actualResponse.Content.Headers);
}
}
}
public async Task HttpRetryHandler_ReturnsContentHeaders()
{
// Arrange
Func <HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
{
var stringContent = new StringContent("Plain text document.", Encoding.UTF8, "text/plain");
stringContent.Headers.TryAddWithoutValidation("X-Content-ID", "49f47c14-c21f-4c1d-9e13-4f5fcf5f8013");
var response = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = stringContent,
};
response.Headers.TryAddWithoutValidation("X-Message-Header", "This isn't on the content.");
return(response);
};
var retryHandler = new HttpRetryHandler();
var testHandler = new HttpRetryTestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(
httpClient,
() => new HttpRequestMessage(HttpMethod.Get, TestUrl));
var log = new TestLogger();
// Act
var actualResponse = await retryHandler.SendAsync(request, log, CancellationToken.None);
// Assert
var actualHeaders = actualResponse
.Content
.Headers
.OrderBy(x => x.Key)
.ToDictionary(x => x.Key, x => x.Value);
Assert.Equal(
new List <string> {
"Content-Type", "X-Content-ID"
},
actualHeaders.Keys.OrderBy(x => x).ToList());
Assert.Equal(actualHeaders["Content-Type"], new[] { "text/plain; charset=utf-8" });
Assert.Equal(actualHeaders["X-Content-ID"], new[] { "49f47c14-c21f-4c1d-9e13-4f5fcf5f8013" });
}
public async Task HttpRetryHandler_AppliesTimeoutToRequestsIndividually()
{
// Arrange
// 20 requests that take 50ms each for a total of 1 second (plus noise).
var requestDuration = TimeSpan.FromMilliseconds(50);
var maxTries = 20;
// Make the request timeout longer than each request duration but less than the total
// duration of all attempts.
var requestTimeout = TimeSpan.FromMilliseconds(500);
int hits = 0;
Func <HttpRequestMessage, CancellationToken, Task <HttpResponseMessage> > handler = async(requestMessage, token) =>
{
hits++;
await Task.Delay(requestDuration);
return(new HttpResponseMessage(HttpStatusCode.InternalServerError));
};
var retryHandler = new HttpRetryHandler();
var testHandler = new TestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = maxTries,
RequestTimeout = requestTimeout,
RetryDelay = TimeSpan.Zero
};
var log = new TestLogger();
// Act
using (await retryHandler.SendAsync(request, log, CancellationToken.None))
{
}
// Assert
Assert.Equal(maxTries, hits);
}
public async Task HttpRetryHandler_MultipleTriesNoSuccess()
{
// Arrange
TimeSpan retryDelay = TimeSpan.Zero;
TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary <string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = MaxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
var hits = 0;
Func <HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
{
hits++;
return(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable));
};
var retryHandler = new HttpRetryHandler(testEnvironmentVariableReader);
var testHandler = new HttpRetryTestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = MaxTries,
RequestTimeout = Timeout.InfiniteTimeSpan,
RetryDelay = retryDelay
};
var log = new TestLogger();
// Act
using (await retryHandler.SendAsync(request, log, CancellationToken.None))
{
}
// Assert
Assert.Equal(MaxTries, hits);
}
private static async Task <T> ThrowsException <T>(ITestServer server) where T : Exception
{
return(await server.ExecuteAsync(async address =>
{
int maxTries = 2;
TimeSpan retryDelay = TimeSpan.Zero;
TestEnvironmentVariableReader testEnvironmentVariableReader = new TestEnvironmentVariableReader(
new Dictionary <string, string>()
{
[EnhancedHttpRetryHelper.RetryCountEnvironmentVariableName] = maxTries.ToString(),
[EnhancedHttpRetryHelper.DelayInMillisecondsEnvironmentVariableName] = retryDelay.TotalMilliseconds.ToString()
});
// Arrange
var retryHandler = new HttpRetryHandler(testEnvironmentVariableReader);
var countingHandler = new CountingHandler {
InnerHandler = new HttpClientHandler()
};
var httpClient = new HttpClient(countingHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, address))
{
MaxTries = maxTries,
RetryDelay = retryDelay
};
// Act
Func <Task> actionAsync = () => retryHandler.SendAsync(
request,
new TestLogger(),
CancellationToken.None);
// Act & Assert
var exception = await Assert.ThrowsAsync <T>(actionAsync);
Assert.Equal(2, countingHandler.Hits);
return exception;
}));
}
public async Task HttpRetryHandler_MultipleTriesTimed()
{
// Arrange
Func <HttpRequestMessage, HttpResponseMessage> handler = requestMessage =>
{
return(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable));
};
var minTime = GetRetryMinTime(MaxTries, SmallTimeout);
var retryHandler = new HttpRetryHandler();
var testHandler = new TestHandler(handler);
var httpClient = new HttpClient(testHandler);
var request = new HttpRetryHandlerRequest(httpClient, () => new HttpRequestMessage(HttpMethod.Get, TestUrl))
{
MaxTries = MaxTries,
RequestTimeout = Timeout.InfiniteTimeSpan,
RetryDelay = SmallTimeout
};
var log = new TestLogger();
// Act
var timer = new Stopwatch();
timer.Start();
using (await retryHandler.SendAsync(request, log, CancellationToken.None))
{
}
timer.Stop();
// Assert
Assert.True(
timer.Elapsed >= minTime,
$"Expected this to take at least: {minTime} But it finished in: {timer.Elapsed}");
}