public static IReadWritableProcessState Clone(this IReadWritableProcessState process, IEntityFactory factory) { var entity = factory.CreateProcessEntity(); entity.ProcessId = process.ProcessId; entity.BranchId = process.BranchId; entity.CompanyId = process.CompanyId; entity.CorrelationId = process.CorrelationId; entity.Criteria = process.Criteria; entity.GroupId = process.GroupId; //process.Id = entity.Id; entity.IsFinished = process.IsFinished; return(entity); //todo }
public static void MarkProcessStatus(this IReadWritableProcessState state, CompletionStatus completionStatus, ResultStatus result, string reason, IStateManager stateManager, IProcessRepository registeredProcesses, IProcessExecutionContext executionContext, IBatchEngineSubscribers batchEngineSubscribers, IFrameworkLogger fLogger) { state.Status = completionStatus; state.Result = result; if (completionStatus.IsDone()) { state.CompleteTime = DateTime.UtcNow; } state.IsFinished = completionStatus.Id == CompletionStatus.Finished.Id; state.IsStopped = completionStatus.Id == CompletionStatus.Stopped.Id; stateManager.SaveProcess(state); state.TriggerProcessEvents(registeredProcesses, executionContext, result.Id == ResultStatus.Error.Id, state.IsStopped, batchEngineSubscribers, fLogger, reason); }
public int MarkProcessForRetry(IReadWritableProcessState process) { throw new NotImplementedException(); }
//void InvokeProcessExecuteComplete(IProcessState state, bool isFailed) //{er // var process = _registeredProcesses.GetRegisteredProcesses().FirstOrDefault(p => p.ProcessKey == state.ProcessKey); // var context = _cacheAside.GetProcessExecutionContext(state.Id); // process?.InvokeProcessCompeteEvent(context, isFailed); //} //void InvokeProcessRetry(IProcessState state) //{ // var process = _registeredProcesses.GetRegisteredProcesses().FirstOrDefault(p => p.ProcessKey == state.ProcessKey); // var context = _cacheAside.GetProcessExecutionContext(state.Id); // process?.InvokeProcessRetry(context); //} bool CheckProcessCompletion(IReadWritableProcessState state) { Logger.Trace($"Process Watchdog triggered for processId {state.Id}"); var processId = state.Id; var configuration = _cacheAside.GetProcessConfiguration(state.ProcessId); var processLogger = _loggerFactory.GetProcessLogger(processId, state.ProcessId, state.CorrelationId); var executionContext = (ProcessExecutionContext)_cacheAside.GetProcessExecutionContext(state.Id); executionContext.UpdateProcessEntity(state); if (!state.IsExecuting()) { Logger.Trace($"Process Watchdog not executing with status IsStopped {state.IsStopped}, IsFinished {state.IsFinished} for processId {processId}"); _eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString());//clean resources return(true); } var timeoutMins = configuration.ProcessTimeoutMins ?? 0; if (!state.StartTime.HasValue) { state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error, $"Process start time is not marked for {processId}", _stateManager, _registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger); //InvokeProcessExecuteComplete(state, true); //moved to extensions _eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString()); return(true); } var incompleteTasks = _stateManager.GetIncompleteTasksCountForProcess(processId); // .GetIncompleteTasksForProcess(processId).ToList(); if (incompleteTasks > 0) { //check timeout var isTimedout = timeoutMins > 0 && state.StartTime.Value.AddMinutes(timeoutMins) < DateTime.UtcNow; if (isTimedout) { processLogger.Error("Timeout"); state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error, $"Process timeout {processId}", _stateManager, _registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger); //InvokeProcessExecuteComplete(state, true);//moved to extensions _eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString()); return(true); } //var deferredTasks = incompleteTasks.Where(d => d.DeferredCount > 0).ToList(); Logger.Trace($"Process Watchdog skipped for processId {processId}. {incompleteTasks} incomplete tasks"); // & {deferredTasks.Count} deferred tasks _eventAggregator.PublishAsync(this, Constants.EventInvokeProducer); //todo publish to other nodes return(false); } else { //check retry configured var erroredTasks = _stateManager.CountFailedTasksForProcess <ITaskState>(processId); void StopGroup(string message) { state.IsStopped = true; state.MarkProcessStatus(CompletionStatus.Stopped, ResultStatus.Error, message, _stateManager, _registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger); if (_runningGroups.TryGetValue(state.GroupId, out SubmittedGroup grp)) { executionContext.Logger.Error(message); _groupsHandler.StopGroup(grp.GroupEntity, message); } else { message = $"ProcessStop not found in running groups => QId {state.Id}, PId {state.ProcessId} => {message}"; _systemLogger.Error(message); } _eventAggregator.Publish(this, Constants.EventProcessStop, processId.ToString()); } bool CheckStopGroup() { var stopNeeded = executionContext.Configuration.ErrorThreshold.HasValue && executionContext.Configuration.ErrorThreshold.Value > 0; if (stopNeeded && erroredTasks >= executionContext.Configuration.ErrorThreshold.Value) { var message = $"ProcessStop QId {state.Id}, PId {state.ProcessId} meets errors threshold {executionContext.Configuration.ErrorThreshold.Value} with errors count {erroredTasks}. Going to stop"; StopGroup(message); return(true); } return(false); } if (erroredTasks > 0) { if (configuration.ProcessRetries.HasValue) { var configuredRetries = configuration.ProcessRetries.Value; if (state.RetryCount < configuredRetries && configuredRetries > 0) { bool retry = state.CanRetryProcess(executionContext, executionContext.Logger, _registeredProcesses, _batchEngineSubscribers, _stateManager, _systemLogger, out bool stop, out string message); if (stop) { StopGroup(message); return(true); } if (!retry) { //state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error, // $"Retry stopped by extension {processSubscriber.GetType()}", stateManager, registeredProcesses, executionContext, batchEngineSubscribers, fLogger); CheckStopGroup(); //_eventAggregator.Publish(this, Constants.EventProcessStop, processId.ToString()); return(true); //process completed } #region commented //ProcessRetryContext context = // new ProcessRetryContext(processId, state.ProcessKey, processLogger); //foreach (var processSubscriber in _batchEngineSubscribers.GetProcessSubscribers()) //{ // InvokeProcessRetry(state); // Robustness.Instance.SafeCall(() => processSubscriber.OnProcessRetry(context), // Logger); // if (context.StopFlag) // { // processLogger.Warn($"Retry stopped by extension {processSubscriber.GetType()}"); // state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error, // $"Retry stopped by extension {processSubscriber.GetType()}", _stateManager, TODO, TODO, TODO, TODO); // InvokeProcessExecuteComplete(state, true); // _eventAggregator.Publish(this, Constants.EventProcessStop, // processId.ToString()); // return true; // } //} #endregion //todo goto retry processLogger.Warn($"Process going to retry with errors {erroredTasks}"); var retryTasksCount = state.ResetProcessTasksForRetry(_stateManager); processLogger.Info($"{retryTasksCount} marked for retry"); _eventAggregator.PublishAsync(this, Constants.EventProcessRetry, processId.ToString()); return(false); //not yet completed } else { //retries complete } } if (CheckStopGroup()) { return(true); } state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error, $"Process completed with errors {erroredTasks}", _stateManager, _registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger); //InvokeProcessExecuteComplete(state, true); //moved to extensions _eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString()); return(true); } else { state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Success, "Process completed", _stateManager, _registeredProcesses, executionContext, _batchEngineSubscribers, _systemLogger); //InvokeProcessExecuteComplete(state, false); ////moved to extensions _eventAggregator.Publish(this, Constants.EventProcessFinished, processId.ToString()); return(true); } } }
public int MarkProcessForRetry(IReadWritableProcessState process) { return(Execute(() => _stateManagerImplementation.MarkProcessForRetry(process))); }
public static bool CanGenerateProcessVolume(this IReadWritableProcessState processState) { return(CompletionStatus.Pending.Id != processState.Status.Id || processState.IsStopped); }
public static int ResetProcessTasksForRetry(this IReadWritableProcessState state, IStateManager steManager) { //todo: update retrycount, start time and reset error tasks //IStateManager steManager = Resolver.Instance.Resolve<IStateManager>(); return(steManager.MarkProcessForRetry(state)); }
static void TriggerProcessEvents(this IReadWritableProcessState state, IProcessRepository registeredProcesses, IProcessExecutionContext context, bool isFailed, bool isStopped, IBatchEngineSubscribers batchEngineSubscribers, IFrameworkLogger fLogger, string reason) { var process = registeredProcesses.GetRegisteredProcesses().FirstOrDefault(p => p.ProcessKey == context.Configuration.ProcessKey); //var context = _cacheAside.GetProcessExecutionContext(state.Id); //process?.InvokeProcessCompeteEvent(context, isFailed); var subscribers = batchEngineSubscribers.GetProcessSubscribers().Where(p => p.ProcessKey == context.Configuration.ProcessKey); //ProcessRetryContext ct =new ProcessRetryContext(context.ProcessState.Id, context.ProcessState.ProcessKey, context.Logger); ProcessCompleteContext ct = new ProcessCompleteContext(context.ProcessState.Id, context.ProcessState.ProcessId, false, false, context.Logger); //IProcessCompleteContext ctx=new proccom ProcessStoppedContext processStoppedContext = null; foreach (var subscriber in subscribers) { if (isFailed) { Robustness.Instance.SafeCall(() => subscriber.ProcessFailed(ct) , fLogger); } else if (isStopped) { if (processStoppedContext == null) { processStoppedContext = new ProcessStoppedContext(state.Id, state.ProcessId, context.Configuration.ProcessKey, reason, context.Logger); } var stoppedContext = processStoppedContext;//access to modified closure Robustness.Instance.SafeCall(() => subscriber.OnProcessStop(stoppedContext) , fLogger); } else { Robustness.Instance.SafeCall(() => subscriber.OnProcessComplete(ct) , fLogger); } } if (isStopped) { Robustness.Instance.SafeCall(() => { if (processStoppedContext == null) { processStoppedContext = new ProcessStoppedContext(state.Id, state.ProcessId, context.Configuration.ProcessKey, reason, context.Logger); } process?.ProcessStopped(processStoppedContext); }); } else if (isFailed) { Robustness.Instance.SafeCall(() => { process?.ProcessFailed(context); }); } else { Robustness.Instance.SafeCall(() => { process?.ProcessCompleted(context); }); } Robustness.Instance.SafeCall(() => { process?.ProcessFinalizer(context); }); }