protected virtual async Task _runOnce(CancellationToken ctk = default)
{
var now = DateTime.UtcNow;
var sw = Stopwatch.StartNew();
MappedDiagnosticsLogicalContext.Set("RequestID", Guid.NewGuid().ToString());
_logger.Info("Check started for tenant {0} at {1}", _config.Tenant, now);
try
{
var infos = await _getResourcesInfo(ctk).ConfigureAwait(false);
var bad = infos.GroupBy(x => x.ResourceId).FirstOrDefault(x => x.Count() > 1);
if (bad != null)
{
throw new InvalidOperationException($"Found multiple entries for ResouceId:{bad.Key}");
}
if (_config.SkipResourcesOlderThanDays.HasValue)
{
infos = infos
.Where(x => x.Modified.Date > LocalDateTime.FromDateTime(now).Date.PlusDays(-(int)_config.SkipResourcesOlderThanDays.Value))
;
}
var list = infos.ToList();
_logger.Info("Found {0} resources in {1}", list.Count, sw.Elapsed);
// check which entries are new or have been modified.
var states = _config.IgnoreState ? Enumerable.Empty <ResourceState>() : await _stateProvider.LoadStateAsync(_config.Tenant, list.Select(i => i.ResourceId).ToArray(), ctk).ConfigureAwait(false);
var toProcess = list.GroupJoin(states, i => i.ResourceId, s => s.ResourceId, (i, s) => new { CurrentInfo = i, Match = s.SingleOrDefault() })
.Where(x => x.Match == null || // new resource
(x.Match.RetryCount > 0 && x.Match.RetryCount <= _config.MaxRetries) || // retry
(x.Match.RetryCount == 0 && x.CurrentInfo.Modified > x.Match.Modified) || // new version
(x.Match.RetryCount > _config.MaxRetries &&
x.CurrentInfo.Modified > x.Match.Modified &&
x.Match.LastEvent + _config.BanDuration < SystemClock.Instance.GetCurrentInstant()) // BAN expired and new version
)
.ToList();
var parallelism = _config.DegreeOfParallelism;
_logger.Info("Found {0} resources to process with parallelism {1}", list.Count, parallelism);
using (SemaphoreSlim throttler = new SemaphoreSlim(initialCount: (int)parallelism))
{
int total = toProcess.Count;
var tasks = toProcess.Select(async(x, i) =>
{
await throttler.WaitAsync(ctk).ConfigureAwait(false);
try
{
await _processEntry(i, total, x.CurrentInfo, x.Match, ctk).ConfigureAwait(false);
}
finally
{
throttler.Release();
}
});
await Task.WhenAll(tasks).ConfigureAwait(false);
}
_logger.Info("Check successful for tenant {0} in {1}", _config.Tenant, sw.Elapsed);
if (sw.Elapsed > _config.RunDurationNotificationLimit)
{
_logger.Fatal("Check for tenant {0} took too much: {1}", _config.Tenant, sw.Elapsed);
}
} catch (Exception ex)
{
_logger.Error(ex, "Check failed for tenant {0} in {1}", _config.Tenant, sw.Elapsed);
throw;
}
}
protected virtual async Task _runOnce(RunType runType, CancellationToken ctk = default)
{
var now = DateTime.UtcNow;
MappedDiagnosticsLogicalContext.Set("RequestID", Guid.NewGuid().ToString());
var activityRun = _diagnosticSource.RunStart(runType, now);
try
{
//GetResources
var activityResource = _diagnosticSource.GetResourcesStart();
var infos = await _getResourcesInfo(ctk).ConfigureAwait(false);
var bad = infos.GroupBy(x => x.ResourceId).FirstOrDefault(x => x.Count() > 1);
if (bad != null)
{
_diagnosticSource.ThrowDuplicateResourceIdRetrived(bad.Key);
}
if (_config.SkipResourcesOlderThanDays.HasValue)
{
infos = infos
.Where(x => _getEarliestModified(x).Date > LocalDateTime.FromDateTime(now).Date.PlusDays(-(int)_config.SkipResourcesOlderThanDays.Value))
;
}
var list = infos.ToList();
_diagnosticSource.GetResourcesSuccessful(activityResource, list.Count);
//Check State - check which entries are new or have been modified.
var activityCheckState = _diagnosticSource.CheckStateStart();
var states = _config.IgnoreState ? Enumerable.Empty <ResourceState>() : await _stateProvider.LoadStateAsync(_config.Tenant, list.Select(i => i.ResourceId).ToArray(), ctk).ConfigureAwait(false);
var evaluated = _createEvalueteList(list, states);
_diagnosticSource.CheckStateSuccessful(activityCheckState, evaluated);
//Process
var skipped = evaluated.Where(x => x.ResultType == ResultType.Skipped).ToList();
var toProcess = evaluated.Where(x => !x.ResultType.HasValue).ToList();
_logger.Info($"Found {skipped.Count} resources to skip");
_logger.Info($"Found {toProcess.Count} resources to process with parallelism {_config.DegreeOfParallelism}");
var count = toProcess.Count;
await toProcess.Parallel((int)_config.DegreeOfParallelism, (i, x, ct) => {
x.Total = count;
x.Index = i + 1;
return(_processEntry(x, ct));
}, ctk);
_diagnosticSource.RunSuccessful(activityRun, evaluated);
if (activityRun.Duration > _config.RunDurationNotificationLimit)
{
_diagnosticSource.RunTookTooLong(activityRun);
}
}
catch (Exception ex)
{
_diagnosticSource.RunFailed(activityRun, ex);
throw;
}
}
