public static IReadWritableProcessState Clone(this IReadWritableProcessState process, IEntityFactory factory)
{
var entity = factory.CreateProcessEntity();
entity.ProcessId = process.ProcessId;
entity.BranchId = process.BranchId;
entity.CompanyId = process.CompanyId;
entity.CorrelationId = process.CorrelationId;
entity.Criteria = process.Criteria;
entity.GroupId = process.GroupId;
//process.Id = entity.Id;
entity.IsFinished = process.IsFinished;
return(entity);
//todo
}
public static void MarkProcessStatus(this IReadWritableProcessState state, CompletionStatus completionStatus,
ResultStatus result, string reason, IStateManager stateManager, IProcessRepository registeredProcesses,
IProcessExecutionContext executionContext, IBatchEngineSubscribers batchEngineSubscribers,
IFrameworkLogger fLogger)
{
state.Status = completionStatus;
state.Result = result;
if (completionStatus.IsDone())
{
state.CompleteTime = DateTime.UtcNow;
}
state.IsFinished = completionStatus.Id == CompletionStatus.Finished.Id;
state.IsStopped = completionStatus.Id == CompletionStatus.Stopped.Id;
stateManager.SaveProcess(state);
state.TriggerProcessEvents(registeredProcesses, executionContext, result.Id == ResultStatus.Error.Id, state.IsStopped,
batchEngineSubscribers, fLogger, reason);
}
public int MarkProcessForRetry(IReadWritableProcessState process)
{
throw new NotImplementedException();
}
//void InvokeProcessExecuteComplete(IProcessState state, bool isFailed)
//{er
// var process = _registeredProcesses.GetRegisteredProcesses().FirstOrDefault(p => p.ProcessKey == state.ProcessKey);
// var context = _cacheAside.GetProcessExecutionContext(state.Id);
// process?.InvokeProcessCompeteEvent(context, isFailed);
//}
//void InvokeProcessRetry(IProcessState state)
//{
// var process = _registeredProcesses.GetRegisteredProcesses().FirstOrDefault(p => p.ProcessKey == state.ProcessKey);
// var context = _cacheAside.GetProcessExecutionContext(state.Id);
// process?.InvokeProcessRetry(context);
//}
bool CheckProcessCompletion(IReadWritableProcessState state)
{
Logger.Trace($"Process Watchdog triggered for processId {state.Id}");
var processId = state.Id;
var configuration = _cacheAside.GetProcessConfiguration(state.ProcessId);
var processLogger = _loggerFactory.GetProcessLogger(processId, state.ProcessId, state.CorrelationId);
var executionContext = (ProcessExecutionContext)_cacheAside.GetProcessExecutionContext(state.Id);
executionContext.UpdateProcessEntity(state);
if (!state.IsExecuting())
{
Logger.Trace($"Process Watchdog not executing with status IsStopped {state.IsStopped}, IsFinished {state.IsFinished} for processId {processId}");
_eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString());//clean resources
return(true);
}
var timeoutMins = configuration.ProcessTimeoutMins ?? 0;
if (!state.StartTime.HasValue)
{
state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error,
$"Process start time is not marked for {processId}", _stateManager,
_registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger);
//InvokeProcessExecuteComplete(state, true); //moved to extensions
_eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString());
return(true);
}
var incompleteTasks = _stateManager.GetIncompleteTasksCountForProcess(processId); // .GetIncompleteTasksForProcess(processId).ToList();
if (incompleteTasks > 0)
{
//check timeout
var isTimedout = timeoutMins > 0 && state.StartTime.Value.AddMinutes(timeoutMins) < DateTime.UtcNow;
if (isTimedout)
{
processLogger.Error("Timeout");
state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error,
$"Process timeout {processId}", _stateManager,
_registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger);
//InvokeProcessExecuteComplete(state, true);//moved to extensions
_eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString());
return(true);
}
//var deferredTasks = incompleteTasks.Where(d => d.DeferredCount > 0).ToList();
Logger.Trace($"Process Watchdog skipped for processId {processId}. {incompleteTasks} incomplete tasks"); // & {deferredTasks.Count} deferred tasks
_eventAggregator.PublishAsync(this, Constants.EventInvokeProducer); //todo publish to other nodes
return(false);
}
else
{
//check retry configured
var erroredTasks = _stateManager.CountFailedTasksForProcess <ITaskState>(processId);
void StopGroup(string message)
{
state.IsStopped = true;
state.MarkProcessStatus(CompletionStatus.Stopped, ResultStatus.Error,
message, _stateManager, _registeredProcesses,
executionContext, _batchEngineSubscribers, _systemLogger);
if (_runningGroups.TryGetValue(state.GroupId, out SubmittedGroup grp))
{
executionContext.Logger.Error(message);
_groupsHandler.StopGroup(grp.GroupEntity, message);
}
else
{
message = $"ProcessStop not found in running groups => QId {state.Id}, PId {state.ProcessId} => {message}";
_systemLogger.Error(message);
}
_eventAggregator.Publish(this, Constants.EventProcessStop, processId.ToString());
}
bool CheckStopGroup()
{
var stopNeeded = executionContext.Configuration.ErrorThreshold.HasValue &&
executionContext.Configuration.ErrorThreshold.Value > 0;
if (stopNeeded && erroredTasks >= executionContext.Configuration.ErrorThreshold.Value)
{
var message =
$"ProcessStop QId {state.Id}, PId {state.ProcessId} meets errors threshold {executionContext.Configuration.ErrorThreshold.Value} with errors count {erroredTasks}. Going to stop";
StopGroup(message);
return(true);
}
return(false);
}
if (erroredTasks > 0)
{
if (configuration.ProcessRetries.HasValue)
{
var configuredRetries = configuration.ProcessRetries.Value;
if (state.RetryCount < configuredRetries && configuredRetries > 0)
{
bool retry = state.CanRetryProcess(executionContext, executionContext.Logger,
_registeredProcesses, _batchEngineSubscribers, _stateManager, _systemLogger, out bool stop, out string message);
if (stop)
{
StopGroup(message);
return(true);
}
if (!retry)
{
//state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error,
// $"Retry stopped by extension {processSubscriber.GetType()}", stateManager, registeredProcesses, executionContext, batchEngineSubscribers, fLogger);
CheckStopGroup();
//_eventAggregator.Publish(this, Constants.EventProcessStop, processId.ToString());
return(true); //process completed
}
#region commented
//ProcessRetryContext context =
// new ProcessRetryContext(processId, state.ProcessKey, processLogger);
//foreach (var processSubscriber in _batchEngineSubscribers.GetProcessSubscribers())
//{
// InvokeProcessRetry(state);
// Robustness.Instance.SafeCall(() => processSubscriber.OnProcessRetry(context),
// Logger);
// if (context.StopFlag)
// {
// processLogger.Warn($"Retry stopped by extension {processSubscriber.GetType()}");
// state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error,
// $"Retry stopped by extension {processSubscriber.GetType()}", _stateManager, TODO, TODO, TODO, TODO);
// InvokeProcessExecuteComplete(state, true);
// _eventAggregator.Publish(this, Constants.EventProcessStop,
// processId.ToString());
// return true;
// }
//}
#endregion
//todo goto retry
processLogger.Warn($"Process going to retry with errors {erroredTasks}");
var retryTasksCount = state.ResetProcessTasksForRetry(_stateManager);
processLogger.Info($"{retryTasksCount} marked for retry");
_eventAggregator.PublishAsync(this, Constants.EventProcessRetry, processId.ToString());
return(false); //not yet completed
}
else
{
//retries complete
}
}
if (CheckStopGroup())
{
return(true);
}
state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error, $"Process completed with errors {erroredTasks}", _stateManager,
_registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger);
//InvokeProcessExecuteComplete(state, true); //moved to extensions
_eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString());
return(true);
}
else
{
state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Success, "Process completed",
_stateManager,
_registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger);
//InvokeProcessExecuteComplete(state, false); ////moved to extensions
_eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString());
return(true);
}
}
}
public int MarkProcessForRetry(IReadWritableProcessState process)
{
return(Execute(() => _stateManagerImplementation.MarkProcessForRetry(process)));
}
public static bool CanGenerateProcessVolume(this IReadWritableProcessState processState)
{
return(CompletionStatus.Pending.Id != processState.Status.Id || processState.IsStopped);
}
public static int ResetProcessTasksForRetry(this IReadWritableProcessState state, IStateManager steManager)
{
//todo: update retrycount, start time and reset error tasks
//IStateManager steManager = Resolver.Instance.Resolve<IStateManager>();
return(steManager.MarkProcessForRetry(state));
}
static void TriggerProcessEvents(this IReadWritableProcessState state, IProcessRepository registeredProcesses, IProcessExecutionContext context,
bool isFailed, bool isStopped, IBatchEngineSubscribers batchEngineSubscribers, IFrameworkLogger fLogger, string reason)
{
var process = registeredProcesses.GetRegisteredProcesses().FirstOrDefault(p => p.ProcessKey == context.Configuration.ProcessKey);
//var context = _cacheAside.GetProcessExecutionContext(state.Id);
//process?.InvokeProcessCompeteEvent(context, isFailed);
var subscribers = batchEngineSubscribers.GetProcessSubscribers().Where(p => p.ProcessKey == context.Configuration.ProcessKey);
//ProcessRetryContext ct =new ProcessRetryContext(context.ProcessState.Id, context.ProcessState.ProcessKey, context.Logger);
ProcessCompleteContext ct = new ProcessCompleteContext(context.ProcessState.Id, context.ProcessState.ProcessId, false, false, context.Logger);
//IProcessCompleteContext ctx=new proccom
ProcessStoppedContext processStoppedContext = null;
foreach (var subscriber in subscribers)
{
if (isFailed)
{
Robustness.Instance.SafeCall(() =>
subscriber.ProcessFailed(ct)
, fLogger);
}
else if (isStopped)
{
if (processStoppedContext == null)
{
processStoppedContext = new ProcessStoppedContext(state.Id, state.ProcessId, context.Configuration.ProcessKey, reason, context.Logger);
}
var stoppedContext = processStoppedContext;//access to modified closure
Robustness.Instance.SafeCall(() =>
subscriber.OnProcessStop(stoppedContext)
, fLogger);
}
else
{
Robustness.Instance.SafeCall(() =>
subscriber.OnProcessComplete(ct)
, fLogger);
}
}
if (isStopped)
{
Robustness.Instance.SafeCall(() =>
{
if (processStoppedContext == null)
{
processStoppedContext = new ProcessStoppedContext(state.Id, state.ProcessId, context.Configuration.ProcessKey, reason, context.Logger);
}
process?.ProcessStopped(processStoppedContext);
});
}
else if (isFailed)
{
Robustness.Instance.SafeCall(() =>
{
process?.ProcessFailed(context);
});
}
else
{
Robustness.Instance.SafeCall(() =>
{
process?.ProcessCompleted(context);
});
}
Robustness.Instance.SafeCall(() =>
{
process?.ProcessFinalizer(context);
});
}
