private async Task markProgress(HighWaterStatistics statistics, TimeSpan delayTime)
{
if (!IsRunning)
{
return;
}
// don't bother sending updates if the current position is 0
if (statistics.CurrentMark == 0 || statistics.CurrentMark == _tracker.HighWaterMark)
{
await Task.Delay(delayTime, _token).ConfigureAwait(false);
return;
}
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.LogDebug("High Water mark detected at {CurrentMark}", statistics.CurrentMark);
}
_current = statistics;
_tracker.MarkHighWater(statistics.CurrentMark);
await Task.Delay(delayTime, _token).ConfigureAwait(false);
}
private async Task <long> findCurrentMark(HighWaterStatistics statistics, IManagedConnection conn, CancellationToken token)
{
// look for the current mark
_start.Value = statistics.SafeStartMark;
using var reader = await conn.ExecuteReaderAsync(_gapDetection, token);
// If there is a row, this tells us the first sequence gap
if (await reader.ReadAsync(token))
{
return(await reader.GetFieldValueAsync <long>(0, token));
}
// use the latest sequence in the event table
await reader.NextResultAsync(token);
if (!await reader.ReadAsync(token))
{
return(statistics.CurrentMark);
}
if (!(await reader.IsDBNullAsync(0, token)))
{
return(await reader.GetFieldValueAsync <long>(0, token));
}
// This happens when the agent is restarted with persisted
// state, and has no previous current mark.
if (statistics.CurrentMark == 0 && statistics.LastMark > 0)
{
return(statistics.LastMark);
}
return(statistics.CurrentMark);
}
private async Task calculateHighWaterMark(HighWaterStatistics statistics, CancellationToken token)
{
// If the last high water mark is the same as the highest number
// assigned from the sequence, then the high water mark cannot
// have changed
if (statistics.LastMark == statistics.HighestSequence)
{
statistics.CurrentMark = statistics.LastMark;
}
else if (statistics.HighestSequence == 0)
{
statistics.CurrentMark = statistics.LastMark = 0;
}
else
{
statistics.CurrentMark = await findCurrentMark(statistics, token);
}
if (statistics.HasChanged)
{
_newSeq.Value = statistics.CurrentMark;
await _runner.SingleCommit(_updateStatus, token);
if (!statistics.LastUpdated.HasValue)
{
var current = await loadCurrentStatistics(token);
statistics.LastUpdated = current.LastUpdated;
}
}
}
private async Task <HighWaterStatistics> loadCurrentStatistics(IManagedConnection conn, CancellationToken token)
{
var statistics = new HighWaterStatistics();
using var reader = await conn.ExecuteReaderAsync(_stateDetection, token);
if (await reader.ReadAsync(token))
{
statistics.HighestSequence = await reader.GetFieldValueAsync <long>(0, token);
}
await reader.NextResultAsync(token);
if (!await reader.ReadAsync(token))
{
return(statistics);
}
statistics.LastMark = statistics.SafeStartMark = await reader.GetFieldValueAsync <long>(0, token);
statistics.LastUpdated = await reader.GetFieldValueAsync <DateTimeOffset>(1, token);
statistics.Timestamp = await reader.GetFieldValueAsync <DateTimeOffset>(2, token);
return(statistics);
}
private async Task markProgress(HighWaterStatistics statistics, TimeSpan delayTime)
{
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.LogDebug("High Water mark detected at " + statistics.CurrentMark);
}
_current = statistics;
_tracker.MarkHighWater(statistics.CurrentMark);
await Task.Delay(delayTime, _token);
}
public HighWaterStatus InterpretStatus(HighWaterStatistics previous)
{
if (HighestSequence == 1 && CurrentMark == 0)
{
return(HighWaterStatus.CaughtUp);
}
if (CurrentMark > previous.CurrentMark)
{
return(CurrentMark == HighestSequence ? HighWaterStatus.CaughtUp : HighWaterStatus.Changed);
}
return(HighWaterStatus.Stale);
}
public async Task Start()
{
_current = await _detector.Detect(_token);
_tracker.Publish(new ShardState(ShardState.HighWaterMark, _current.CurrentMark)
{
Action = ShardAction.Started
});
_loop = Task.Factory.StartNew(DetectChanges, TaskCreationOptions.LongRunning | TaskCreationOptions.AttachedToParent);
_timer.Start();
_logger.LogInformation("Started HighWaterAgent");
}
private async Task <long> findCurrentMark(HighWaterStatistics statistics, CancellationToken token)
{
// look for the current mark
_gapDetector.Start = statistics.SafeStartMark;
var current = await _runner.Query(_gapDetector, token);
if (current.HasValue)
{
return(current.Value);
}
// This happens when the agent is restarted with persisted
// state, and has no previous current mark.
if (statistics.CurrentMark == 0 && statistics.LastMark > 0)
{
return(statistics.LastMark);
}
return(statistics.CurrentMark);
}
public HighWaterStatus InterpretStatus(HighWaterStatistics previous)
{
// Postgres sequences start w/ 1 by default. So the initial state is "HighestSequence = 1".
if (HighestSequence == 1 && CurrentMark == 0)
{
return(HighWaterStatus.CaughtUp);
}
if (CurrentMark == HighestSequence)
{
return(HighWaterStatus.CaughtUp);
}
if (CurrentMark > previous.CurrentMark)
{
return(HighWaterStatus.Changed);
}
return(HighWaterStatus.Stale);
}
private async Task calculateHighWaterMark(CancellationToken token, HighWaterStatistics statistics,
IManagedConnection conn)
{
// If the last high water mark is the same as the highest number
// assigned from the sequence, then the high water mark cannot
// have changed
if (statistics.LastMark == statistics.HighestSequence)
{
statistics.CurrentMark = statistics.LastMark;
}
else if (statistics.HighestSequence == 0)
{
statistics.CurrentMark = statistics.LastMark = 0;
}
else
{
statistics.CurrentMark = await findCurrentMark(statistics, conn, token);
}
if (statistics.HasChanged)
{
await conn.BeginTransactionAsync(token);
_newSeq.Value = statistics.CurrentMark;
await conn.ExecuteAsync(_updateStatus, token);
await conn.CommitAsync(token);
if (!statistics.LastUpdated.HasValue)
{
var current = await loadCurrentStatistics(conn, token);
statistics.LastUpdated = current.LastUpdated;
}
}
}
private async Task DetectChanges()
{
// TODO -- need to put some retry & exception handling here.
try
{
_current = await _detector.Detect(_token);
if (_current.CurrentMark > 0)
{
_tracker.MarkHighWater(_current.CurrentMark);
}
}
catch (Exception e)
{
_logger.LogError(e, "Failed while making the initial determination of the high water mark");
}
await Task.Delay(_settings.FastPollingTime, _token);
while (!_token.IsCancellationRequested)
{
HighWaterStatistics statistics = null;
try
{
statistics = await _detector.Detect(_token);
}
catch (Exception e)
{
_logger.LogError(e, "Failed while trying to detect high water statistics");
await Task.Delay(_settings.SlowPollingTime, _token);
continue;
}
var status = statistics.InterpretStatus(_current);
switch (status)
{
case HighWaterStatus.Changed:
await markProgress(statistics, _settings.FastPollingTime);
break;
case HighWaterStatus.CaughtUp:
await markProgress(statistics, _settings.SlowPollingTime);
break;
case HighWaterStatus.Stale:
var safeHarborTime = _current.Timestamp.Add(_settings.StaleSequenceThreshold);
var delayTime = safeHarborTime.Subtract(statistics.Timestamp);
if (delayTime.TotalSeconds > 0)
{
await Task.Delay(delayTime, _token);
}
statistics = await _detector.DetectInSafeZone(safeHarborTime, _token);
await markProgress(statistics, _settings.FastPollingTime);
break;
}
}
_logger.LogInformation("HighWaterAgent has detected a cancellation and has stopped polling");
}
private async Task DetectChanges()
{
if (!IsRunning)
{
return;
}
try
{
_current = await _detector.Detect(_token).ConfigureAwait(false);
if (_current.CurrentMark > 0)
{
_tracker.MarkHighWater(_current.CurrentMark);
}
}
catch (Exception e)
{
_logger.LogError(e, "Failed while making the initial determination of the high water mark");
}
await Task.Delay(_settings.FastPollingTime, _token).ConfigureAwait(false);
while (!_token.IsCancellationRequested)
{
if (!IsRunning)
{
break;
}
HighWaterStatistics statistics = null;
try
{
statistics = await _detector.Detect(_token).ConfigureAwait(false);
}
catch (Exception e)
{
_logger.LogError(e, "Failed while trying to detect high water statistics");
await Task.Delay(_settings.SlowPollingTime, _token).ConfigureAwait(false);
continue;
}
var status = statistics.InterpretStatus(_current);
switch (status)
{
case HighWaterStatus.Changed:
await markProgress(statistics, _settings.FastPollingTime).ConfigureAwait(false);
break;
case HighWaterStatus.CaughtUp:
await markProgress(statistics, _settings.SlowPollingTime).ConfigureAwait(false);
break;
case HighWaterStatus.Stale:
_logger.LogInformation("High Water agent is stale at {CurrentMark}", statistics.CurrentMark);
// This gives the high water detection a chance to allow the gaps to fill in
// before skipping to the safe harbor time
var safeHarborTime = _current.Timestamp.Add(_settings.StaleSequenceThreshold);
if (safeHarborTime > statistics.Timestamp)
{
await Task.Delay(_settings.SlowPollingTime, _token).ConfigureAwait(false);
continue;
}
_logger.LogInformation("High Water agent is stale after threshold of {DelayInSeconds} seconds, skipping gap to events marked after {SafeHarborTime}", _settings.StaleSequenceThreshold.TotalSeconds, safeHarborTime);
statistics = await _detector.DetectInSafeZone(_token).ConfigureAwait(false);
await markProgress(statistics, _settings.FastPollingTime).ConfigureAwait(false);
break;
}
}
_logger.LogInformation("HighWaterAgent has detected a cancellation and has stopped polling");
}
