public Activity ProcessResourceStart(ProcessContext processContext)
{
bool result = processContext.IsResourceUpdated(out var infos);
if (!result)
{
_logger.Info("No changes detected on ResourceId=\"{0}\""
, processContext.CurrentInfo.ResourceId
);
}
else
{
_logger.Info("({4}/{5}) Detected change on ResourceId=\"{0}\", Resource.ModifiedSource={6},Resource.Modified={1}, OldState.Modified={2}, OldState.Retry={3}. Processing..."
, processContext.CurrentInfo.ResourceId
, infos.current?.ToString()
, infos.last?.ToString()
, processContext.LastState?.RetryCount
, processContext.Index
, processContext.Total
, infos.source ?? string.Empty
);
}
Activity activity = _start("ProcessResource", () => new
{
ProcessContext = processContext,
Tenant = _tenant,
});
return(activity);
}
public void ProcessResourceSuccessful(Activity activity, int idx, int total, ProcessContext processContext)
{
//_setTags(activity, processType.ToString(), processType.ToString());
_stop(activity, () => new
{
Idx = idx,
Total = total,
ProcessContext = processContext,
Tenant = _tenant,
}
);
if (processContext.ResultType == ResultType.NoNewData)
{
_logger.Info($"({idx}/{total}) ResourceId=\"{processContext.CurrentInfo.ResourceId}\" No payload retrived, so no new state. Generally due to a same-checksum");
}
else if (processContext.ResultType == ResultType.NoAction)
{
_logger.Info($"({idx}/{total}) ResourceId=\"{processContext.CurrentInfo.ResourceId}\" No action has been triggered and payload has not been retrieved. We do not change the state");
}
else if (processContext.ResultType == ResultType.Normal)
{
_logger.Info($"({idx}/{total}) ResourceId=\"{processContext.CurrentInfo.ResourceId}\" handled {(processContext.NewState.RetryCount == 0 ? "" : "not ")}successfully {(activity != null ? " in" + (activity.Duration.ToString()) : "")}");
}
}
public void FetchResourceFailed(Activity activity, ProcessContext pc, Exception ex)
{
_stop(activity, () => new
{
ProcessContext = pc,
Exception = ex,
Tenant = _tenant,
}
);
}
public void FetchResourceSuccessful(Activity activity, ProcessContext pc)
{
//_setTags(activity, processType.ToString(), processType.ToString());
_stop(activity, () => new
{
ProcessContext = pc,
Tenant = _tenant,
}
);
}
public Activity FetchResourceStart(ProcessContext pc)
{
Activity activity = _start("FetchResource", () => new
{
ProcessContext = pc,
Tenant = _tenant,
}
);
return(activity);
}
public void ProcessResourceFailed(Activity activity, ProcessContext pc, bool isBanned, Exception ex)
{
var lvl = isBanned ? LogLevel.Fatal : LogLevel.Warn;
_logger.Log(lvl, ex, $"({pc.Index}/{pc.Total}) ResourceId=\"{pc.CurrentInfo.ResourceId}\" process Failed");
_stop(activity, () => new
{
ProcessContext = pc,
Exception = ex,
Tenant = _tenant,
});
}
private List <ProcessContext> _createEvalueteList(List <IResourceMetadata> list, IEnumerable <ResourceState> states)
{
var ev = list.GroupJoin(states, i => i.ResourceId, s => s.ResourceId, (i, s) =>
{
var x = new ProcessContext {
CurrentInfo = i, LastState = s.SingleOrDefault()
};
if (x.LastState == null)
{
x.ProcessType = ProcessType.New;
}
else if (x.LastState.RetryCount == 0 && x.IsResourceUpdated(out _))
{
x.ProcessType = ProcessType.Updated;
}
else if (x.LastState.RetryCount > 0 && x.LastState.RetryCount <= _config.MaxRetries)
{
x.ProcessType = ProcessType.Retry;
}
else if (x.LastState.RetryCount > _config.MaxRetries &&
x.IsResourceUpdated(out _) &&
x.LastState.LastEvent + _config.BanDuration < SystemClock.Instance.GetCurrentInstant()
// BAN expired and new version
)
{
x.ProcessType = ProcessType.RetryAfterBan;
}
else if (x.LastState.RetryCount > _config.MaxRetries &&
x.IsResourceUpdated(out _) &&
!(x.LastState.LastEvent + _config.BanDuration < SystemClock.Instance.GetCurrentInstant())
// BAN
)
{
x.ProcessType = ProcessType.Banned;
}
else
{
x.ProcessType = ProcessType.NothingToDo;
}
if (x.ProcessType == ProcessType.NothingToDo || x.ProcessType == ProcessType.Banned)
{
x.ResultType = ResultType.Skipped;
}
return(x);
}).ToList();
return(ev);
}
public void ProcessResourceFailed(Activity activity, int idx, int total, ProcessContext processContext, bool isBanned, Exception ex)
{
var lvl = isBanned ? LogLevel.Fatal : LogLevel.Warn;
_logger.Log(lvl, ex, $"({idx}/{total}) ResourceId=\"{processContext.CurrentInfo.ResourceId}\" process Failed");
_stop(activity, () => new
{
Idx = idx,
Total = total,
ProcessContext = processContext,
IsBanned = isBanned,
Exception = ex,
Tenant = _tenant,
}
);
}
private async Task <T> _fetchResource(ProcessContext pc, CancellationToken ctk = default)
{
var info = pc.CurrentInfo;
var lastState = pc.LastState;
var activity = _diagnosticSource.FetchResourceStart(pc);
try
{
var res = await _retrievePayload(info, lastState, ctk);
_diagnosticSource.FetchResourceSuccessful(activity, pc);
return(res);
} catch (Exception ex)
{
_diagnosticSource.FetchResourceFailed(activity, pc, ex);
throw;
}
}
public Activity ProcessResourceStart(ProcessContext processContext)
{
_logger.Info("({4}/{5}) Detected change on ResourceId=\"{0}\", Resource.Modified={1}, OldState.Modified={2}, OldState.Retry={3}. Processing..."
, processContext.CurrentInfo.ResourceId
, processContext.CurrentInfo.Modified
, processContext.LastState?.Modified
, processContext.LastState?.RetryCount
, processContext.Index
, processContext.Total
);
Activity activity = _start("ProcessResource", () => new
{
ProcessContext = processContext,
Tenant = _tenant,
});
return(activity);
}
private async Task _processEntry(ProcessContext pc, CancellationToken ctk = default)
{
var info = pc.CurrentInfo;
var lastState = pc.LastState;
var dataType = pc.ProcessType;
try
{
var processActivity = _diagnosticSource.ProcessResourceStart(pc);
AsyncLazy <T> payload = new AsyncLazy <T>(() => _fetchResource(pc, ctk));
var state = pc.NewState = new ResourceState()
{
Tenant = _config.Tenant,
ResourceId = info.ResourceId,
Modified = lastState?.Modified ?? default, // we want to update modified only on success or Ban or first run
ModifiedSources = lastState?.ModifiedSources, // we want to update modified multiple only on success or Ban or first run
LastEvent = SystemClock.Instance.GetCurrentInstant(),
RetryCount = lastState?.RetryCount ?? 0,
CheckSum = lastState?.CheckSum,
RetrievedAt = lastState?.RetrievedAt,
Extensions = info.Extensions
};
try
{
pc.ResultType = ResultType.Normal;
IResourceState newState = default;
await _processResource(new ChangedStateContext <T>(info, lastState, payload), ctk).ConfigureAwait(false);
// if handlers retrived data, fetch the result to check the checksum
if (payload.IsStarted)
{
// if the retrievePayload task gone in exception but the _processResource did not ...
// here we care only if we have a payload to use
if (payload.Task.Status == TaskStatus.RanToCompletion)
{
newState = await payload;
if (newState != null)
{
if (!string.IsNullOrWhiteSpace(newState.CheckSum) && state.CheckSum != newState.CheckSum)
{
_logger.Info("Checksum changed on ResourceId=\"{0}\" from \"{1}\" to \"{2}\"", state.ResourceId, state.CheckSum, newState.CheckSum);
}
state.CheckSum = newState.CheckSum;
state.RetrievedAt = newState.RetrievedAt;
pc.ResultType = ResultType.Normal;
}
else // no payload retrived, so no new state. Generally due to a same-checksum
{
pc.ResultType = ResultType.NoNewData;
}
state.Extensions = info.Extensions;
state.Modified = info.Modified;
state.ModifiedSources = info.ModifiedSources;
state.RetryCount = 0; // success
}
else
{
throw new NotSupportedException($"({pc.Index}/{pc.Total}) ResourceId=\"{state.ResourceId}\" we cannot reach this point!");
}
}
else // for some reason, no action has been and payload has not been retrieved. We do not change the state
{
pc.ResultType = ResultType.NoAction;
}
_diagnosticSource.ProcessResourceSuccessful(processActivity, pc);
if (processActivity.Duration > _config.ResourceDurationNotificationLimit)
{
_diagnosticSource.ProcessResourceTookTooLong(info.ResourceId, processActivity);
}
}
catch (Exception ex)
{
state.LastException = ex;
pc.ResultType = ResultType.Error;
var isBanned = ++state.RetryCount == _config.MaxRetries;
state.Extensions = info.Extensions;
_diagnosticSource.ProcessResourceFailed(processActivity, pc, isBanned, ex);
}
await _stateProvider.SaveStateAsync(new[] { state }, ctk).ConfigureAwait(false);
}
catch (Exception ex)
{
// chomp it, we'll retry this file next time, forever, fuckit
pc.ResultType = ResultType.Error;
_diagnosticSource.ProcessResourceSaveFailed(info.ResourceId, ex);
}
}
public virtual void OnProcessResourceStop(string tenant, int idx, int total, ProcessContext processContext, bool isBanned, Exception exception)
{
}
public virtual void OnProcessResourceStop(string tenant, ProcessContext processContext, Exception exception)
{
}
