public void GetDocumentIsSuccesfull()
{
var identity = new TestIdentity();
var state0 = new TestState()
{
Val = "Initial"
};
var savedState = stateProvider.SaveStateAsync(CancellationToken.None, identity, state0).Result; //blocking on a task for test purposes, should have a timeout
var readState = stateProvider.GetStateAsync(CancellationToken.None, identity).Result;
Assert.True(savedState.ETag == readState.ETag && savedState.Val == readState.Val && savedState.Val != null);
}
private async Task _processEntry(int idx, int total, IResourceMetadata info, ResourceState lastState, CancellationToken ctk = default)
{
try
{
_logger.Info("({4}/{5}) Detected change on Resource.Id=\"{0}\", Resource.Modified={1}, OldState.Modified={2}, OldState.Retry={3}. Processing..."
, info.ResourceId
, info.Modified
, lastState?.Modified
, lastState?.RetryCount
, idx
, total
);
var sw = Stopwatch.StartNew();
AsyncLazy <T> payload = new AsyncLazy <T>(() => _retrievePayload(info, lastState, ctk));
var state = new ResourceState()
{
Tenant = _config.Tenant,
ResourceId = info.ResourceId,
Modified = lastState?.Modified ?? info.Modified, // we want to update modified only on success or Ban or first run
LastEvent = SystemClock.Instance.GetCurrentInstant(),
RetryCount = lastState?.RetryCount ?? 0,
CheckSum = lastState?.CheckSum,
RetrievedAt = lastState?.RetrievedAt,
Extensions = info.Extensions
};
try {
await _processResource(new ChangedStateContext <T>(info, lastState, payload), ctk).ConfigureAwait(false);
// if handlers retrived data, fetch the result to check the checksum
if (payload.IsStarted)
{
// if the retrievePayload task gone in exception but the _processResource did not ...
// here we care only if we have a payload to use
if (payload.Task.Status == TaskStatus.RanToCompletion)
{
var newState = await payload;
if (newState != null)
{
if (!string.IsNullOrWhiteSpace(newState.CheckSum) && state.CheckSum != newState.CheckSum)
{
_logger.Info("Checksum changed on Resource.Id=\"{0}\" from \"{1}\" to \"{2}\"", state.ResourceId, state.CheckSum, newState.CheckSum);
}
state.CheckSum = newState.CheckSum;
state.RetrievedAt = newState.RetrievedAt;
}
else // no payload retrived, so no new state. Generally due to a same-checksum
{
}
state.Modified = info.Modified;
state.RetryCount = 0; // success
}
}
else // for some reason, no action has been and payload has not been retrieved. We do not change the state
{
}
}
catch (Exception ex)
{
state.LastException = ex;
LogLevel lvl = ++state.RetryCount == _config.MaxRetries ? LogLevel.Fatal : LogLevel.Warn;
_logger.Log(lvl, ex, "Error while processing ResourceId=\"{0}\"", info.ResourceId);
// if we're in BAN and we tried to exit BAN and we failed, update the Modified anw
if (state.RetryCount > _config.MaxRetries)
{
state.Modified = info.Modified;
}
}
await _stateProvider.SaveStateAsync(new[] { state }, ctk).ConfigureAwait(false);
_logger.Info("({3}/{4}) ResourceId=\"{0}\" handled {2}successfully in {1}", state.ResourceId, sw.Elapsed, state.RetryCount == 0 ? "" : "not ", idx
, total);
if (sw.Elapsed > _config.ResourceDurationNotificationLimit)
{
_logger.Fatal("Processing of ResourceId=\"{0}\" took too much: {1}", state.ResourceId, sw.Elapsed);
}
} catch (Exception ex)
{
// chomp it, we'll retry this file next time, forever, fuckit
_logger.Error(ex, "({4}/{5}) Error in processing the ResourceId=\"{0}\". The execution will be automatically retried.", info.ResourceId);
}
}
private async Task _processEntry(ProcessContext pc, CancellationToken ctk = default)
{
var info = pc.CurrentInfo;
var lastState = pc.LastState;
var dataType = pc.ProcessType;
try
{
var processActivity = _diagnosticSource.ProcessResourceStart(pc);
AsyncLazy <T> payload = new AsyncLazy <T>(() => _fetchResource(pc, ctk));
var state = pc.NewState = new ResourceState()
{
Tenant = _config.Tenant,
ResourceId = info.ResourceId,
Modified = lastState?.Modified ?? default, // we want to update modified only on success or Ban or first run
ModifiedSources = lastState?.ModifiedSources, // we want to update modified multiple only on success or Ban or first run
LastEvent = SystemClock.Instance.GetCurrentInstant(),
RetryCount = lastState?.RetryCount ?? 0,
CheckSum = lastState?.CheckSum,
RetrievedAt = lastState?.RetrievedAt,
Extensions = info.Extensions
};
try
{
pc.ResultType = ResultType.Normal;
IResourceState newState = default;
await _processResource(new ChangedStateContext <T>(info, lastState, payload), ctk).ConfigureAwait(false);
// if handlers retrived data, fetch the result to check the checksum
if (payload.IsStarted)
{
// if the retrievePayload task gone in exception but the _processResource did not ...
// here we care only if we have a payload to use
if (payload.Task.Status == TaskStatus.RanToCompletion)
{
newState = await payload;
if (newState != null)
{
if (!string.IsNullOrWhiteSpace(newState.CheckSum) && state.CheckSum != newState.CheckSum)
{
_logger.Info("Checksum changed on ResourceId=\"{0}\" from \"{1}\" to \"{2}\"", state.ResourceId, state.CheckSum, newState.CheckSum);
}
state.CheckSum = newState.CheckSum;
state.RetrievedAt = newState.RetrievedAt;
pc.ResultType = ResultType.Normal;
}
else // no payload retrived, so no new state. Generally due to a same-checksum
{
pc.ResultType = ResultType.NoNewData;
}
state.Extensions = info.Extensions;
state.Modified = info.Modified;
state.ModifiedSources = info.ModifiedSources;
state.RetryCount = 0; // success
}
else
{
throw new NotSupportedException($"({pc.Index}/{pc.Total}) ResourceId=\"{state.ResourceId}\" we cannot reach this point!");
}
}
else // for some reason, no action has been and payload has not been retrieved. We do not change the state
{
pc.ResultType = ResultType.NoAction;
}
_diagnosticSource.ProcessResourceSuccessful(processActivity, pc);
if (processActivity.Duration > _config.ResourceDurationNotificationLimit)
{
_diagnosticSource.ProcessResourceTookTooLong(info.ResourceId, processActivity);
}
}
catch (Exception ex)
{
state.LastException = ex;
pc.ResultType = ResultType.Error;
var isBanned = ++state.RetryCount == _config.MaxRetries;
state.Extensions = info.Extensions;
_diagnosticSource.ProcessResourceFailed(processActivity, pc, isBanned, ex);
}
await _stateProvider.SaveStateAsync(new[] { state }, ctk).ConfigureAwait(false);
}
catch (Exception ex)
{
// chomp it, we'll retry this file next time, forever, fuckit
pc.ResultType = ResultType.Error;
_diagnosticSource.ProcessResourceSaveFailed(info.ResourceId, ex);
}
}
