private async Task WaitForBeaconShutdownAsync(TimeBudget budget)
{
try
{
log.Info("Service beacon graceful deregistration has been initiated (up to {ServiceBeaconWaitTime}).", budget.Remaining);
var replicaInfo = serviceBeacon.ReplicaInfo;
var elapsedBefore = budget.Elapsed;
while (!budget.HasExpired)
{
var topology = serviceLocator.Locate(replicaInfo.Environment, replicaInfo.Application);
var replica = topology?.Replicas.FirstOrDefault(r => r.ToString().Equals(replicaInfo.Replica, StringComparison.OrdinalIgnoreCase));
if (replica == null)
{
log.Info("Service replica has disappeared from topology according to local service locator in {ServiceBeaconWaitDuration}.", budget.Elapsed - elapsedBefore);
break;
}
await Task.Delay(TimeSpanArithmetics.Min(budget.Remaining, 100.Milliseconds())).ConfigureAwait(false);
}
// (iloktionov): The rest of the wait is a safety net (other applications may receive SD notifications significantly later).
await Task.Delay(budget.Remaining).ConfigureAwait(false);
}
catch (Exception error)
{
log.Error(error, "Failed to wait gracefully for service beacon deregistration.");
}
}
/// <inheritdoc />
public async Task SendAsync(Request request, RequestParameters parameters, IRequestSender sender, IRequestTimeBudget budget, IEnumerable <Uri> replicas, int replicasCount, CancellationToken cancellationToken)
{
var currentReplicaIndex = 0;
foreach (var replica in replicas)
{
if (budget.HasExpired)
{
break;
}
if (request.ContainsAlreadyUsedStream())
{
break;
}
var timeout = TimeSpanArithmetics.Min(timeoutsProvider.GetTimeout(request, budget, currentReplicaIndex++, replicasCount), budget.Remaining);
var connectionAttemptTimeout = currentReplicaIndex == replicasCount ? null : parameters.ConnectionTimeout;
var result = await sender.SendToReplicaAsync(replica, request, connectionAttemptTimeout, timeout, cancellationToken).ConfigureAwait(false);
if (result.Verdict == ResponseVerdict.Accept)
{
break;
}
cancellationToken.ThrowIfCancellationRequested();
}
}
private Window <T, TKey> CreateWindow(T @event, DateTimeOffset timestamp, StreamCoordinates coordinates)
{
var period = TimeSpanArithmetics.Min(settings.PeriodProvider?.Invoke(@event) ?? settings.Period, settings.MaximumAllowedPeriod);
var lag = TimeSpanArithmetics.Min(settings.LagProvider?.Invoke(@event) ?? settings.Lag, settings.MaximumAllowedLag);
var start = timestamp.AddTicks(-timestamp.Ticks % period.Ticks);
var result = new Window <T, TKey>(settings.CreateWindow(key), coordinates, start, start + period, period, lag);
return(result);
}
/// <inheritdoc />
public TimeSpan GetTimeout(Request request, IRequestTimeBudget budget, int currentReplicaIndex, int totalReplicas)
{
if (currentReplicaIndex >= providers.Length)
{
return(tailBehaviour == TailTimeoutBehaviour.UseRemainingBudget
? budget.Remaining
: TimeSpanArithmetics.Min(providers.Last()(), budget.Remaining));
}
return(TimeSpanArithmetics.Min(providers[currentReplicaIndex](), budget.Remaining));
}
/// <inheritdoc />
public TimeSpan GetRetryDelay(int attemptsUsed)
{
var delay = InitialRetryDelay + RetryDelayIncrement.Multiply(Math.Max(0, attemptsUsed - 1));
var jitterAmount = delay.Multiply(ThreadSafeRandom.NextDouble() * Jitter);
if (ThreadSafeRandom.NextDouble() <= 0.5)
{
jitterAmount = jitterAmount.Negate();
}
return(TimeSpanArithmetics.Min(MaximumRetryDelay, delay + jitterAmount));
}
public static (HostingShutdown hosting, ApplicationShutdown application) Create(
IServiceBeacon serviceBeacon,
IServiceLocator serviceLocator,
IVostokApplicationIdentity identity,
IMetricContext instanceMetrics,
ILog log,
int?port,
IReadOnlyList <CancellationToken> tokens,
TimeSpan totalTimeout,
TimeSpan beaconTimeout,
bool beaconWaitEnabled,
bool sendAnnotation)
{
var hasRealBeacon = serviceBeacon is ServiceBeacon;
var hasRealLocator = serviceLocator is ServiceLocator;
// (iloktionov): No point in waiting for beacon deregistration for apps without external port or when SD is disabled.
beaconWaitEnabled &= port.HasValue && hasRealBeacon && hasRealLocator;
// (iloktionov): No point in reducing app shutdown timeout right from the start when SD is disabled.
beaconTimeout = hasRealBeacon ? TimeSpanArithmetics.Min(beaconTimeout, totalTimeout.Divide(3)) : TimeSpan.Zero;
// (iloktionov): Artificially reduce initial app shutdown timeout by beacon shutdown timeout so that it's value doesn't drop abruptly on shutdown.
var applicationShutdown = new ApplicationShutdown(log, totalTimeout - beaconTimeout);
var hostingToken = tokens.Any() ? CancellationTokenSource.CreateLinkedTokenSource(tokens.ToArray()).Token : default;
var hostingShutdown = new HostingShutdown(
applicationShutdown,
serviceBeacon,
serviceLocator,
identity,
instanceMetrics,
log,
hostingToken,
totalTimeout,
beaconTimeout,
beaconWaitEnabled,
sendAnnotation);
return(hostingShutdown, applicationShutdown);
}
public void Min_should_return_minimum_value_of_given_three()
{
TimeSpanArithmetics.Min(1.Seconds(), 2.Seconds(), 3.Seconds()).Should().Be(1.Seconds());
TimeSpanArithmetics.Min(3.Seconds(), 1.Seconds(), 2.Seconds()).Should().Be(1.Seconds());
TimeSpanArithmetics.Min(2.Seconds(), 3.Seconds(), 1.Seconds()).Should().Be(1.Seconds());
}