public async Task ManyWaitsStackUp()
{
await Policy
.Handle <InRangeException>()
.RetryAsync(3)
.ExecuteAsync(async() =>
{
var limiter = new Limiter(new Limit(10), 5);
using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5));
while (cancellation.IsCancellationRequested == false)
{
var task = limiter.WaitAsync(cancellation.Token);
if (!task.IsCompleted)
{
await task.ConfigureAwait(false);
break;
}
}
var delayOne = new Stopwatch();
delayOne.Start();
var delayTwo = new Stopwatch();
delayTwo.Start();
var delayThree = new Stopwatch();
delayThree.Start();
var waits = new List <Task>
{
limiter.WaitAsync(cancellation.Token),
limiter.WaitAsync(cancellation.Token),
limiter.WaitAsync(cancellation.Token),
};
var taskOne = await Task.WhenAny(waits).ConfigureAwait(false);
await taskOne.ConfigureAwait(false);
delayOne.Stop();
waits.Remove(taskOne);
var taskTwo = await Task.WhenAny(waits).ConfigureAwait(false);
await taskTwo.ConfigureAwait(false);
delayTwo.Stop();
waits.Remove(taskTwo);
var taskThree = await Task.WhenAny(waits).ConfigureAwait(false);
await taskThree.ConfigureAwait(false);
delayThree.Stop();
waits.Remove(taskThree);
Assert.InRange(delayOne.Elapsed, TimeSpan.FromMilliseconds(75), TimeSpan.FromMilliseconds(125));
Assert.InRange(delayTwo.Elapsed, TimeSpan.FromMilliseconds(175), TimeSpan.FromMilliseconds(225));
Assert.InRange(delayThree.Elapsed, TimeSpan.FromMilliseconds(275), TimeSpan.FromMilliseconds(325));
}).ConfigureAwait(false);
}
public async Task WaitAsyncCausesPauseLikeReserve()
{
var limiter = new Limiter(new Limit(10), 5);
using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5));
while (cancellation.IsCancellationRequested == false)
{
var task = limiter.WaitAsync(cancellation.Token);
if (!task.IsCompleted)
{
await task.ConfigureAwait(false);
break;
}
}
var delayOne = new Stopwatch();
delayOne.Start();
await limiter.WaitAsync(cancellation.Token).ConfigureAwait(false);
delayOne.Stop();
var delayTwoMore = new Stopwatch();
delayTwoMore.Start();
await limiter.WaitAsync(2, cancellation.Token).ConfigureAwait(false);
delayTwoMore.Stop();
await Task.Delay(TimeSpan.FromMilliseconds(150)).ConfigureAwait(false);
var delayAlreadyAvailable = new Stopwatch();
delayAlreadyAvailable.Start();
await limiter.WaitAsync(cancellation.Token).ConfigureAwait(false);
delayAlreadyAvailable.Stop();
var delayHalfAvailable = new Stopwatch();
delayHalfAvailable.Start();
await limiter.WaitAsync(cancellation.Token).ConfigureAwait(false);
delayHalfAvailable.Stop();
Assert.InRange(delayOne.Elapsed, TimeSpan.FromMilliseconds(75), TimeSpan.FromMilliseconds(125));
Assert.InRange(delayTwoMore.Elapsed, TimeSpan.FromMilliseconds(175), TimeSpan.FromMilliseconds(225));
Assert.InRange(delayAlreadyAvailable.Elapsed, TimeSpan.Zero, TimeSpan.FromMilliseconds(5));
Assert.InRange(delayHalfAvailable.Elapsed, TimeSpan.FromMilliseconds(25), TimeSpan.FromMilliseconds(75));
}
public async Task WaitAsync_Should_StopTimeDelayOnCancel()
{
using var cts = new CancellationTokenSource();
var limiter = new Limiter(1, TimeSpan.FromSeconds(10), SystemTime.Instance);
await limiter.WaitAsync(cts.Token).ConfigureAwait(false);
var task = limiter.WaitAsync(cts.Token);
cts.Cancel();
await Assert.ThrowsAsync <OperationCanceledException>(async() => await task.ConfigureAwait(false)).ConfigureAwait(false);
}
public async Task WaitAsync_Should_StopLockingOnCancel()
{
using var cts = new CancellationTokenSource();
var time = A.Fake <ITime>();
var date = new DateTimeOffset(new DateTime(2000, 1, 1), default);
A.CallTo(() => time.UtcNow).Returns(date);
A.CallTo(() => time.Delay(A <TimeSpan> ._, A <CancellationToken> ._))
.ReturnsLazily(call => Task.Delay(20_000, (CancellationToken)call.Arguments[1]));
var limiter = new Limiter(1, TimeSpan.FromSeconds(10), time);
await limiter.WaitAsync(cts.Token).ConfigureAwait(false);
var waitTaskOnDelay = limiter.WaitAsync(cts.Token);
var taskWaitOnLock = limiter.WaitAsync(cts.Token);
cts.Cancel();
await Assert.ThrowsAsync <OperationCanceledException>(async() => await taskWaitOnLock.ConfigureAwait(false)).ConfigureAwait(false);
}
public override async Task RunAsync(CancellationToken cancellationToken)
{
using var client = new HttpClient();
while (!cancellationToken.IsCancellationRequested)
{
await _limiter.WaitAsync(cancellationToken).ConfigureAwait(false);
#pragma warning disable CA1303 // Do not pass literals as localized parameters
Logger.LogInformation("Connecting with {ControllerUrl}", _options.ControllerUrl.ToString());
try
{
#if NET
using var stream = await client.GetStreamAsync(_options.ControllerUrl, cancellationToken).ConfigureAwait(false);
#else
using var stream = await client.GetStreamAsync(_options.ControllerUrl).ConfigureAwait(false);
#endif
using var reader = new StreamReader(stream, Encoding.UTF8, leaveOpen: true);
using var cancellation = cancellationToken.Register(stream.Close);
while (true)
{
var json = await reader.ReadLineAsync().ConfigureAwait(false);
if (string.IsNullOrEmpty(json))
{
break;
}
var message = System.Text.Json.JsonSerializer.Deserialize <Message>(json);
Logger.LogInformation("Received {MessageType} for {MessageKey}", message.MessageType, message.Key);
Logger.LogInformation(json);
Logger.LogInformation(message.MessageType.ToString());
if (message.MessageType == MessageType.Update)
{
_proxyConfigProvider.Update(message.Routes, message.Cluster);
}
}
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception ex)
#pragma warning restore CA1031 // Do not catch general exception types
{
Logger.LogInformation("Stream ended: {Message}", ex.Message);
}
#pragma warning restore CA1303 // Do not pass literals as localized parameters
}
}
public async Task WaitAsync_Should_ReturnWithDelayIfSlotsAreFilled()
{
int maxCount = 5;
var interval = TimeSpan.FromMinutes(3);
var time = A.Fake <ITime>();
var date = new DateTimeOffset(new DateTime(2000, 1, 1), default);
A.CallTo(() => time.UtcNow).Returns(date);
var limiter = new Limiter(maxCount, interval, time);
var tasks = Enumerable.Range(0, maxCount).Select(x => limiter.WaitAsync());
var delay = Task.Delay(1_000);
if (delay == await Task.WhenAny(delay, Task.WhenAll(tasks)).ConfigureAwait(false))
{
throw new TimeoutException("Limiter too slow and didn't return slots");
}
await limiter.WaitAsync().ConfigureAwait(false);
A.CallTo(() => time.UtcNow).MustHaveHappened(maxCount + 1, Times.Exactly);
A.CallTo(() => time.Delay(A <TimeSpan> .That.IsEqualTo(interval), A <CancellationToken> ._)).MustHaveHappenedOnceExactly();
}
public async Task WaitAsync_Should_ReturnWithoutDelayIfSlotsAreEmpty()
{
int maxCount = 3;
var interval = TimeSpan.FromMinutes(3);
var time = A.Fake <ITime>();
A.CallTo(() => time.UtcNow).Returns(new DateTimeOffset(new DateTime(2000, 1, 1), default));
var limiter = new Limiter(maxCount, interval, time);
await limiter.WaitAsync().ConfigureAwait(false);
A.CallTo(() => time.UtcNow).MustHaveHappenedOnceExactly();
A.CallTo(() => time.Delay(A <TimeSpan> ._, A <CancellationToken> ._)).MustNotHaveHappened();
}
public override async Task RunAsync(CancellationToken cancellationToken)
{
using var client = new HttpClient();
while (!cancellationToken.IsCancellationRequested)
{
await _limiter.WaitAsync(cancellationToken).ConfigureAwait(false);
Logger.LogInformation($"Connecting with { _options.ControllerUrl.ToString()}");
try
{
using var stream = await client.GetStreamAsync(_options.ControllerUrl, cancellationToken).ConfigureAwait(false);
using var reader = new StreamReader(stream, Encoding.UTF8, leaveOpen: true);
using var cancellation = cancellationToken.Register(stream.Close);
while (true)
{
var json = await reader.ReadLineAsync().ConfigureAwait(false);
if (string.IsNullOrEmpty(json))
{
break;
}
var message = JsonSerializer.Deserialize <Message>(json);
Logger.LogInformation($"Received {message.MessageType} for { message.Key}");
Logger.LogInformation(json);
Logger.LogInformation(message.MessageType.ToString());
if (message.MessageType == MessageType.Update)
{
_proxyConfigProvider.Update(message.Routes, message.Cluster);
}
}
}
catch (Exception ex)
{
Logger.LogInformation($"Stream ended: { ex.Message}");
}
}
}
/// <summary>
/// RunAsync starts processing when StartAsync is called, and is terminated when
/// StopAsync is called.
/// </summary>
/// <param name="cancellationToken">The cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
/// <returns>A <see cref="Task"/> representing the result of the asynchronous operation.</returns>
public override async Task RunAsync(CancellationToken cancellationToken)
{
try
{
var limiter = new Limiter(new Limit(0.2), 3);
var shouldSync = true;
var firstSync = true;
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
if (shouldSync)
{
await ListAsync(cancellationToken).ConfigureAwait(true);
shouldSync = false;
}
if (firstSync)
{
_ready.Release();
firstSync = false;
}
await WatchAsync(cancellationToken).ConfigureAwait(true);
}
catch (IOException ex) when(ex.InnerException is SocketException)
{
Logger.LogDebug(
EventId(EventType.ReceivedError),
"Received error watching {ResourceType}: {ErrorMessage}",
typeof(TResource).Name,
ex.Message);
}
catch (KubernetesException ex)
{
Logger.LogDebug(
EventId(EventType.ReceivedError),
"Received error watching {ResourceType}: {ErrorMessage}",
typeof(TResource).Name,
ex.Message);
// deal with this non-recoverable condition "too old resource version"
// with a re-sync to listing everything again ensuring no subscribers miss updates
if (ex is KubernetesException kubernetesError)
{
if (string.Equals(kubernetesError.Status.Reason, "Expired", StringComparison.Ordinal))
{
shouldSync = true;
}
}
}
// rate limiting the reconnect loop
await limiter.WaitAsync(cancellationToken).ConfigureAwait(true);
}
}
catch (Exception error)
{
Logger.LogInformation(
EventId(EventType.WatchingComplete),
error,
"No longer watching {ResourceType} resources from API server.",
typeof(TResource).Name);
throw;
}
}
