private ProcessConsumer BuildProcessConsumer(long processId, int processKey, long groupId)
{
Logger.Trace($"Process consumer build start for processId {processId}");
var executionContext = _cacheAside.GetProcessExecutionContext(processId);
var taskHandler = _processRepository.GetProcessTaskHandler(executionContext.Configuration.ProcessKey);//processKey
ProcessConsumer consumer = new ProcessConsumer(_token, processKey, processId, groupId, _stateManager, _frameworkLogger, taskHandler,
_processRepository.GetSerializer(taskHandler), Bus, _frameworkLogger, _resolver.Resolve <ITaskListenerHandler>(), executionContext);
consumer.SweeperAction = Trigger;
consumer.Start();
consumer.Completion.ContinueWith(c =>
{
Logger.Trace($"Process consumer for processId {processId} stopped and removing from storage. Faulted {c.IsFaulted}, Cancel {c.IsCanceled}, completed {c.IsCompleted}");
_processConsumer.TryRemove(processId, out ProcessConsumer cns);
//todo publish consumer complete notification
consumer.SweeperAction = null;
Robustness.Instance.SafeCall(() =>
{
consumer.Dispose();
}, Logger);
});
Logger.Trace($"Process consumer created processId {processId}");
return(consumer);
}
private void Run()
{
var taskItem = _volumeHandler.GetNextTaskWithTransaction(out ITransaction transaction);
while (taskItem != null && Interrupter?.IsCancellationRequested == false)
{
SafeTransactionWrapper transactionWrapper = null;
try
{
transactionWrapper = new SafeTransactionWrapper(transaction);
var processExecutionContext = _cacheAside.GetProcessExecutionContext(taskItem.ProcessId);
if (processExecutionContext == null)
{
Logger.Error($"TaskPicker: Process configuration not found for Process {taskItem?.ProcessId} against task {taskItem.Id.ToString()} ");
transactionWrapper.Rollback();
transactionWrapper.Dispose();
continue;
}
//var processKey = processExecutionContext.ProcessState.ProcessKey;
var logger = _batchLoggerFactory.GetTaskLogger(taskItem.Id, taskItem.ProcessId, processExecutionContext.ProcessState.CorrelationId);
TaskMessage taskMessage = new TaskMessage(taskItem, transaction, new SafeDisposableActions(transactionWrapper.Rollback), logger);
taskMessage.ProcessContext = processExecutionContext;
logger.Info($"Task picked at Node {NodeSettings.Instance.Name}");
Bus.HandleTaskMessage(taskMessage);
}
catch (Exception e)
{
Logger.Error($"Error picking task {taskItem?.Id.ToString()} for Process {taskItem?.ProcessId} with message {e.Message}", e);
transactionWrapper?.Dispose();
}
taskItem = _volumeHandler.GetNextTaskWithTransaction(out transaction);
}
transaction?.Dispose();
}
//void InvokeProcessExecuteComplete(IProcessState state, bool isFailed)
//{er
// var process = _registeredProcesses.GetRegisteredProcesses().FirstOrDefault(p => p.ProcessKey == state.ProcessKey);
// var context = _cacheAside.GetProcessExecutionContext(state.Id);
// process?.InvokeProcessCompeteEvent(context, isFailed);
//}
//void InvokeProcessRetry(IProcessState state)
//{
// var process = _registeredProcesses.GetRegisteredProcesses().FirstOrDefault(p => p.ProcessKey == state.ProcessKey);
// var context = _cacheAside.GetProcessExecutionContext(state.Id);
// process?.InvokeProcessRetry(context);
//}
bool CheckProcessCompletion(IReadWritableProcessState state)
{
Logger.Trace($"Process Watchdog triggered for processId {state.Id}");
var processId = state.Id;
var configuration = _cacheAside.GetProcessConfiguration(state.ProcessId);
var processLogger = _loggerFactory.GetProcessLogger(processId, state.ProcessId, state.CorrelationId);
var executionContext = (ProcessExecutionContext)_cacheAside.GetProcessExecutionContext(state.Id);
executionContext.UpdateProcessEntity(state);
if (!state.IsExecuting())
{
Logger.Trace($"Process Watchdog not executing with status IsStopped {state.IsStopped}, IsFinished {state.IsFinished} for processId {processId}");
_eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString());//clean resources
return(true);
}
var timeoutMins = configuration.ProcessTimeoutMins ?? 0;
if (!state.StartTime.HasValue)
{
state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error,
$"Process start time is not marked for {processId}", _stateManager,
_registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger);
//InvokeProcessExecuteComplete(state, true); //moved to extensions
_eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString());
return(true);
}
var incompleteTasks = _stateManager.GetIncompleteTasksCountForProcess(processId); // .GetIncompleteTasksForProcess(processId).ToList();
if (incompleteTasks > 0)
{
//check timeout
var isTimedout = timeoutMins > 0 && state.StartTime.Value.AddMinutes(timeoutMins) < DateTime.UtcNow;
if (isTimedout)
{
processLogger.Error("Timeout");
state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error,
$"Process timeout {processId}", _stateManager,
_registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger);
//InvokeProcessExecuteComplete(state, true);//moved to extensions
_eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString());
return(true);
}
//var deferredTasks = incompleteTasks.Where(d => d.DeferredCount > 0).ToList();
Logger.Trace($"Process Watchdog skipped for processId {processId}. {incompleteTasks} incomplete tasks"); // & {deferredTasks.Count} deferred tasks
_eventAggregator.PublishAsync(this, Constants.EventInvokeProducer); //todo publish to other nodes
return(false);
}
else
{
//check retry configured
var erroredTasks = _stateManager.CountFailedTasksForProcess <ITaskState>(processId);
void StopGroup(string message)
{
state.IsStopped = true;
state.MarkProcessStatus(CompletionStatus.Stopped, ResultStatus.Error,
message, _stateManager, _registeredProcesses,
executionContext, _batchEngineSubscribers, _systemLogger);
if (_runningGroups.TryGetValue(state.GroupId, out SubmittedGroup grp))
{
executionContext.Logger.Error(message);
_groupsHandler.StopGroup(grp.GroupEntity, message);
}
else
{
message = $"ProcessStop not found in running groups => QId {state.Id}, PId {state.ProcessId} => {message}";
_systemLogger.Error(message);
}
_eventAggregator.Publish(this, Constants.EventProcessStop, processId.ToString());
}
bool CheckStopGroup()
{
var stopNeeded = executionContext.Configuration.ErrorThreshold.HasValue &&
executionContext.Configuration.ErrorThreshold.Value > 0;
if (stopNeeded && erroredTasks >= executionContext.Configuration.ErrorThreshold.Value)
{
var message =
$"ProcessStop QId {state.Id}, PId {state.ProcessId} meets errors threshold {executionContext.Configuration.ErrorThreshold.Value} with errors count {erroredTasks}. Going to stop";
StopGroup(message);
return(true);
}
return(false);
}
if (erroredTasks > 0)
{
if (configuration.ProcessRetries.HasValue)
{
var configuredRetries = configuration.ProcessRetries.Value;
if (state.RetryCount < configuredRetries && configuredRetries > 0)
{
bool retry = state.CanRetryProcess(executionContext, executionContext.Logger,
_registeredProcesses, _batchEngineSubscribers, _stateManager, _systemLogger, out bool stop, out string message);
if (stop)
{
StopGroup(message);
return(true);
}
if (!retry)
{
//state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error,
// $"Retry stopped by extension {processSubscriber.GetType()}", stateManager, registeredProcesses, executionContext, batchEngineSubscribers, fLogger);
CheckStopGroup();
//_eventAggregator.Publish(this, Constants.EventProcessStop, processId.ToString());
return(true); //process completed
}
#region commented
//ProcessRetryContext context =
// new ProcessRetryContext(processId, state.ProcessKey, processLogger);
//foreach (var processSubscriber in _batchEngineSubscribers.GetProcessSubscribers())
//{
// InvokeProcessRetry(state);
// Robustness.Instance.SafeCall(() => processSubscriber.OnProcessRetry(context),
// Logger);
// if (context.StopFlag)
// {
// processLogger.Warn($"Retry stopped by extension {processSubscriber.GetType()}");
// state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error,
// $"Retry stopped by extension {processSubscriber.GetType()}", _stateManager, TODO, TODO, TODO, TODO);
// InvokeProcessExecuteComplete(state, true);
// _eventAggregator.Publish(this, Constants.EventProcessStop,
// processId.ToString());
// return true;
// }
//}
#endregion
//todo goto retry
processLogger.Warn($"Process going to retry with errors {erroredTasks}");
var retryTasksCount = state.ResetProcessTasksForRetry(_stateManager);
processLogger.Info($"{retryTasksCount} marked for retry");
_eventAggregator.PublishAsync(this, Constants.EventProcessRetry, processId.ToString());
return(false); //not yet completed
}
else
{
//retries complete
}
}
if (CheckStopGroup())
{
return(true);
}
state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error, $"Process completed with errors {erroredTasks}", _stateManager,
_registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger);
//InvokeProcessExecuteComplete(state, true); //moved to extensions
_eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString());
return(true);
}
else
{
state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Success, "Process completed",
_stateManager,
_registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger);
//InvokeProcessExecuteComplete(state, false); ////moved to extensions
_eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString());
return(true);
}
}
}