private async Task WaitForBeaconShutdownAsync(TimeBudget budget)
{
try
{
log.Info("Service beacon graceful deregistration has been initiated (up to {ServiceBeaconWaitTime}).", budget.Remaining);
var replicaInfo = serviceBeacon.ReplicaInfo;
var elapsedBefore = budget.Elapsed;
while (!budget.HasExpired)
{
var topology = serviceLocator.Locate(replicaInfo.Environment, replicaInfo.Application);
var replica = topology?.Replicas.FirstOrDefault(r => r.ToString().Equals(replicaInfo.Replica, StringComparison.OrdinalIgnoreCase));
if (replica == null)
{
log.Info("Service replica has disappeared from topology according to local service locator in {ServiceBeaconWaitDuration}.", budget.Elapsed - elapsedBefore);
break;
}
await Task.Delay(TimeSpanArithmetics.Min(budget.Remaining, 100.Milliseconds())).ConfigureAwait(false);
}
// (iloktionov): The rest of the wait is a safety net (other applications may receive SD notifications significantly later).
await Task.Delay(budget.Remaining).ConfigureAwait(false);
}
catch (Exception error)
{
log.Error(error, "Failed to wait gracefully for service beacon deregistration.");
}
}
/// <inheritdoc />
public async Task SendAsync(Request request, RequestParameters parameters, IRequestSender sender, IRequestTimeBudget budget, IEnumerable <Uri> replicas, int replicasCount, CancellationToken cancellationToken)
{
var currentReplicaIndex = 0;
foreach (var replica in replicas)
{
if (budget.HasExpired)
{
break;
}
if (request.ContainsAlreadyUsedStream())
{
break;
}
var timeout = TimeSpanArithmetics.Min(timeoutsProvider.GetTimeout(request, budget, currentReplicaIndex++, replicasCount), budget.Remaining);
var connectionAttemptTimeout = currentReplicaIndex == replicasCount ? null : parameters.ConnectionTimeout;
var result = await sender.SendToReplicaAsync(replica, request, connectionAttemptTimeout, timeout, cancellationToken).ConfigureAwait(false);
if (result.Verdict == ResponseVerdict.Accept)
{
break;
}
cancellationToken.ThrowIfCancellationRequested();
}
}
public DateTimeOffset?ScheduleNext(DateTimeOffset from)
{
if (!delayFirstIteration && scheduledFirst.TrySetTrue())
{
return(from);
}
return(lastIterationEnd + TimeSpanArithmetics.Max(TimeSpan.Zero, periodProvider()));
}
private async Task <IScheduler> WaitForNextExecutionAsync(DateTimeOffset lastExecutionTime, CancellationToken token)
{
var nextExecutionTime = null as DateTimeOffset?;
var nextExecutionScheduler = null as IScheduler;
var firstActualizationDone = false;
while (!token.IsCancellationRequested)
{
var(newNextExecutionTime, newNextExecutionScheduler) = GetNextExecutionTime(lastExecutionTime);
if (newNextExecutionTime != nextExecutionTime || !firstActualizationDone)
{
nextExecutionTime = newNextExecutionTime;
nextExecutionScheduler = newNextExecutionScheduler;
LogNextExecutionTime(nextExecutionTime);
monitor.OnNextExecution(nextExecutionTime);
}
firstActualizationDone = true;
if (nextExecutionTime == null)
{
await Task.Delay(action.Options.ActualizationPeriod, token);
continue;
}
if (nextExecutionTime <= lastExecutionTime)
{
return(nextExecutionScheduler);
}
var actualizationPeriod = action.Options.ActualizationPeriod;
var timeToWait = TimeSpanArithmetics.Max(TimeSpan.Zero, nextExecutionTime.Value - PreciseDateTime.Now);
if (timeToWait > actualizationPeriod)
{
await Task.Delay(actualizationPeriod, token);
continue;
}
if (timeToWait > TimeSpan.Zero)
{
await Task.Delay(timeToWait, token);
}
while (PreciseDateTime.Now < nextExecutionTime)
{
await Task.Delay(1.Milliseconds(), token);
}
return(nextExecutionScheduler);
}
return(null);
}
private Window <T, TKey> CreateWindow(T @event, DateTimeOffset timestamp, StreamCoordinates coordinates)
{
var period = TimeSpanArithmetics.Min(settings.PeriodProvider?.Invoke(@event) ?? settings.Period, settings.MaximumAllowedPeriod);
var lag = TimeSpanArithmetics.Min(settings.LagProvider?.Invoke(@event) ?? settings.Lag, settings.MaximumAllowedLag);
var start = timestamp.AddTicks(-timestamp.Ticks % period.Ticks);
var result = new Window <T, TKey>(settings.CreateWindow(key), coordinates, start, start + period, period, lag);
return(result);
}
/// <inheritdoc />
public TimeSpan GetTimeout(Request request, IRequestTimeBudget budget, int currentReplicaIndex, int totalReplicas)
{
if (currentReplicaIndex >= divisionFactor)
{
return(budget.Remaining);
}
var effectiveDivisionFactor = Math.Min(divisionFactor, totalReplicas) - currentReplicaIndex;
return(TimeSpanArithmetics.Max(TimeSpan.Zero, budget.Remaining.Divide(effectiveDivisionFactor)));
}
/// <inheritdoc />
public TimeSpan GetTimeout(Request request, IRequestTimeBudget budget, int currentReplicaIndex, int totalReplicas)
{
if (currentReplicaIndex >= providers.Length)
{
return(tailBehaviour == TailTimeoutBehaviour.UseRemainingBudget
? budget.Remaining
: TimeSpanArithmetics.Min(providers.Last()(), budget.Remaining));
}
return(TimeSpanArithmetics.Min(providers[currentReplicaIndex](), budget.Remaining));
}
private void LogNextExecutionTime(DateTimeOffset?nextExecutionTime)
{
if (nextExecutionTime == null)
{
log.Info("Next execution time: unknown.");
}
else
{
log.Info("Next execution time = {NextExecutionTime:yyyy-MM-dd HH:mm:ss.fff} (~{TimeToNextExecution} from now).",
nextExecutionTime.Value.DateTime, TimeSpanArithmetics.Max(TimeSpan.Zero, nextExecutionTime.Value - PreciseDateTime.Now).ToPrettyString());
}
}
/// <inheritdoc />
public TimeSpan GetRetryDelay(int attemptsUsed)
{
var delay = InitialRetryDelay + RetryDelayIncrement.Multiply(Math.Max(0, attemptsUsed - 1));
var jitterAmount = delay.Multiply(ThreadSafeRandom.NextDouble() * Jitter);
if (ThreadSafeRandom.NextDouble() <= 0.5)
{
jitterAmount = jitterAmount.Negate();
}
return(TimeSpanArithmetics.Min(MaximumRetryDelay, delay + jitterAmount));
}
public static (HostingShutdown hosting, ApplicationShutdown application) Create(
IServiceBeacon serviceBeacon,
IServiceLocator serviceLocator,
IVostokApplicationIdentity identity,
IMetricContext instanceMetrics,
ILog log,
int?port,
IReadOnlyList <CancellationToken> tokens,
TimeSpan totalTimeout,
TimeSpan beaconTimeout,
bool beaconWaitEnabled,
bool sendAnnotation)
{
var hasRealBeacon = serviceBeacon is ServiceBeacon;
var hasRealLocator = serviceLocator is ServiceLocator;
// (iloktionov): No point in waiting for beacon deregistration for apps without external port or when SD is disabled.
beaconWaitEnabled &= port.HasValue && hasRealBeacon && hasRealLocator;
// (iloktionov): No point in reducing app shutdown timeout right from the start when SD is disabled.
beaconTimeout = hasRealBeacon ? TimeSpanArithmetics.Min(beaconTimeout, totalTimeout.Divide(3)) : TimeSpan.Zero;
// (iloktionov): Artificially reduce initial app shutdown timeout by beacon shutdown timeout so that it's value doesn't drop abruptly on shutdown.
var applicationShutdown = new ApplicationShutdown(log, totalTimeout - beaconTimeout);
var hostingToken = tokens.Any() ? CancellationTokenSource.CreateLinkedTokenSource(tokens.ToArray()).Token : default;
var hostingShutdown = new HostingShutdown(
applicationShutdown,
serviceBeacon,
serviceLocator,
identity,
instanceMetrics,
log,
hostingToken,
totalTimeout,
beaconTimeout,
beaconWaitEnabled,
sendAnnotation);
return(hostingShutdown, applicationShutdown);
}
/// <inheritdoc />
public void ModifyWeight(HealthWithDecay health, ref double weight)
{
var healthDamage = MaximumHealthValue - health.Value;
if (healthDamage <= 0.0)
{
return;
}
var timeSinceDecayPivot = TimeSpanArithmetics.Max(getCurrentTime() - health.DecayPivot, TimeSpan.Zero);
if (timeSinceDecayPivot >= DecayDuration)
{
return;
}
var effectiveHealth = health.Value + healthDamage * ((double)timeSinceDecayPivot.Ticks / DecayDuration.Ticks);
weight *= effectiveHealth;
}
public ScheduledActionStatistics BuildStatistics()
{
lock (sync)
return(new ScheduledActionStatistics(
currentlyExecuting,
currentlyExecuting ? PreciseDateTime.Now - lastStart.Value : TimeSpan.Zero,
nextExecution.HasValue ? TimeSpanArithmetics.Max(nextExecution.Value - PreciseDateTime.Now, TimeSpan.Zero) : null as TimeSpan?,
nextExecution,
lastStart,
lastFinish,
lastError,
lastErrorMessage,
lastIterationSuccessful,
lastDuration,
totalDuration,
averageDuration,
iterationsStarted,
iterationsSucceeded,
iterationsFailed,
iterationsCompleted));
}
public DateTimeOffset?ScheduleNext(DateTimeOffset from)
{
if (!delayFirstIteration && scheduledFirst.TrySetTrue())
{
return(from);
}
var period = periodProvider();
var jitter = jitterProvider();
period = TimeSpanArithmetics.Max(TimeSpan.Zero, period);
jitter = Math.Max(0d, jitter);
jitter = Math.Min(1d, jitter);
var next = from + period;
if (jitter > 0d)
{
next += period.Multiply(ThreadSafeRandom.NextDouble() * jitter * (ThreadSafeRandom.FlipCoin() ? 1d : -1d));
}
return(next);
}
private async Task ExecutePayloadAsync(DateTimeOffset executionTime, IScheduler scheduler, CancellationToken token)
{
if (token.IsCancellationRequested)
{
return;
}
var nextExecution = GetNextExecutionTime(executionTime).time;
var timeBudget = nextExecution.HasValue
? TimeBudget.StartNew(TimeSpanArithmetics.Max(TimeSpan.Zero, nextExecution.Value - executionTime))
: TimeBudget.Infinite;
var context = new ScheduledActionContext(executionTime, timeBudget, scheduler, token);
if (!(scheduler is PeriodicalWithConstantPauseScheduler))
{
log.Info("Executing with time budget = {TimeBudget}.", timeBudget.Total.ToPrettyString());
}
else
{
log.Info("Executing..");
}
async Task ExecutePayload()
{
monitor.OnIterationStarted();
try
{
var watch = Stopwatch.StartNew();
await action.Payload(context);
watch.Stop();
log.Info(
"Executed in {ExecutionTime}.",
new
{
ExecutionTime = watch.Elapsed.ToPrettyString(),
ExecutionTimeMs = watch.Elapsed.TotalMilliseconds
});
if (watch.Elapsed > timeBudget.Total && !(scheduler is PeriodicalWithConstantPauseScheduler))
{
log.Warn("Execution did not fit into the time budget before the next planned execution.");
}
action.Scheduler.OnSuccessfulIteration(scheduler);
monitor.OnIterationSucceeded();
}
catch (Exception error)
{
action.Scheduler.OnFailedIteration(scheduler, error);
monitor.OnIterationFailed(error);
if (action.Options.CrashOnPayloadException || error is OperationCanceledException)
{
throw;
}
log.Error(error, "Scheduled action threw an exception.");
}
finally
{
monitor.OnIterationCompleted();
}
}
var payloadTask = action.Options.PreferSeparateThread
? Task.Factory.StartNew(ExecutePayload, TaskCreationOptions.LongRunning)
: Task.Run(ExecutePayload);
if (action.Options.AllowOverlappingExecution)
{
return;
}
await payloadTask;
}
public void Max_should_return_minimum_value_of_given_three()
{
TimeSpanArithmetics.Max(1.Seconds(), 2.Seconds(), 3.Seconds()).Should().Be(3.Seconds());
TimeSpanArithmetics.Max(3.Seconds(), 1.Seconds(), 2.Seconds()).Should().Be(3.Seconds());
TimeSpanArithmetics.Max(2.Seconds(), 3.Seconds(), 1.Seconds()).Should().Be(3.Seconds());
}
public void Min_should_return_minimum_value_of_given_two()
{
TimeSpanArithmetics.Min(1.Seconds(), 2.Seconds()).Should().Be(1.Seconds());
TimeSpanArithmetics.Min(2.Seconds(), 1.Seconds()).Should().Be(1.Seconds());
}
