protected void SetWorkflowMessageState(IWorkflowMessage workflowMessage, CancellationToken cancellationToken)
{
if (!WorkflowMessageStates.ContainsKey(workflowMessage.WorkflowMessageId))
{
WorkflowMessageStates[workflowMessage.WorkflowMessageId] = workflowMessage.State;
}
}
Task <WorkflowProcessingResult> IWorkflowEngine.ProcessMessage(IWorkflowMessage workflowMessage)
{
using (var cancellationTokenSource = new CancellationTokenSource())
{
return(((IWorkflowEngine)this).ProcessMessage(workflowMessage, cancellationTokenSource.Token));
}
}
async Task <WorkflowProcessingResult> IWorkflowEngine.ProcessMessage(IWorkflowMessage workflowMessage, CancellationToken cancellationToken)
{
var started = DateTime.UtcNow;
var stopwatch = Stopwatch.StartNew();
WorkflowProcessingResult workflowProcessingResult = null;
try
{
switch (workflowMessage)
{
case IEventRequestWorkflowMessage eventWorkflowMessage:
workflowProcessingResult = await ProcessMessageInternal(eventWorkflowMessage, cancellationToken).ConfigureAwait(false);
break;
case IAsynchronousTransitionWorkflowMessage asynchronousTransitionWorkflowMessage:
workflowProcessingResult = await ProcessMessageInternal(asynchronousTransitionWorkflowMessage, cancellationToken).ConfigureAwait(false);
break;
case IDelayedTransitionWorkflowMessage delayedTransitionWorkflowMessage:
workflowProcessingResult = await ProcessMessageInternal(delayedTransitionWorkflowMessage, cancellationToken).ConfigureAwait(false);
break;
default:
workflowProcessingResult = await ProcessMessageInternal(workflowMessage, cancellationToken).ConfigureAwait(false);
break;
}
return(workflowProcessingResult);
}
finally
{
stopwatch.Stop();
if (Log.IsEnabled(LogEventLevel.Verbose))
{
Log.Verbose("Finished processing of a {messageID}, {duration}, {workflowInstanceId}",
workflowMessage.WorkflowMessageId, stopwatch.Elapsed, workflowProcessingResult?.WorkflowInstance?.Id);
}
if (null != workflowProcessingResult?.WorkflowInstance)
{
// 1. save workflow message execution log into store
var workflowInstanceMessageLog = new WorkflowInstanceMessageLog(workflowProcessingResult.WorkflowInstance.Id, workflowMessage.WorkflowMessageId,
workflowMessage.GetType().Name, started, (int)stopwatch.ElapsedMilliseconds);
await WorkflowEngineBuilder.WorkflowStore.SaveWorkflowInstanceMessageLog(workflowInstanceMessageLog, cancellationToken).ConfigureAwait(false);
// 2. make sure that workflow instance lock is removed
if (workflowProcessingResult.WorkflowInstance.Lock.LockedAt < workflowProcessingResult.WorkflowInstance.Lock.LockedUntil)
{
await UnlockWorkflowInstance(workflowProcessingResult.WorkflowInstance, cancellationToken).ConfigureAwait(false);
}
}
}
}
private async Task <WorkflowProcessingResult> ProcessMessageInternal(IWorkflowMessage workflowMessage, CancellationToken cancellationToken)
{
// TODO: this need to be removed
Guard.ArgumentNotNull(workflowMessage, nameof(workflowMessage));
Guard.ArgumentNotNull(workflowMessage.WorkflowInstanceId, nameof(workflowMessage.WorkflowInstanceId));
Debug.Assert(workflowMessage.WorkflowInstanceId != null, "workflowMessage.WorkflowInstanceId != null");
var workflowInstance = await GetWorkflowInstanceWithLock(workflowMessage.WorkflowInstanceId.Value, cancellationToken).ConfigureAwait(false);
if (null == workflowInstance)
{
throw new WorkflowException($"Workflow message [{workflowMessage.WorkflowMessageId:D}] is referring to non existing workflow instance");
}
var workflowConfiguration = await GetWorkflowConfiguration(workflowInstance.WorkflowId, cancellationToken).ConfigureAwait(false);
Guard.ArgumentNotNull(workflowConfiguration, nameof(workflowConfiguration));
var stateConfiguration = workflowConfiguration.GetStateConfigurationByCode(workflowInstance.CurrentStateCode);
Guard.ArgumentNotNull(stateConfiguration, nameof(stateConfiguration));
var workflowContext = new WorkflowContext(this, WorkflowEngineBuilder, workflowInstance, workflowConfiguration);
// ensure that domain entity is available for all/any coded logic for read-only purposes
await TryGetDomainEntity(workflowContext, cancellationToken).ConfigureAwait(false);
var messageExecutionContext = new MessageExecutionContext(workflowContext, workflowMessage);
var messageExecutionResult = await _messageProcessorLazy.Value.ProcessMessage(messageExecutionContext, cancellationToken).ConfigureAwait(false);
if (messageExecutionResult.Status == MessageExecutionStatus.Failed)
{
Log.Error("An error has occurred during processing message [{messageId}] for {workflowInstanceId}",
workflowMessage.WorkflowMessageId, workflowInstance.Id);
await TransitionToState(workflowContext, workflowConfiguration.GetFailedStateConfiguration(), null, cancellationToken).ConfigureAwait(false);
return(new WorkflowProcessingResult(workflowInstance, workflowContext.WorkflowExecutionState));
}
if (messageExecutionResult.Status == MessageExecutionStatus.ContinueAfterAsyncTransition)
{
if (workflowInstance.CurrentStateProgress != StateExecutionProgress.AwaitingAsyncTransition)
{
Log.Error("Can not process message [{messageId}] for {workflowInstanceId} because workflow instance is not in [{subState}]",
workflowMessage.WorkflowMessageId, workflowInstance.Id, StateExecutionProgress.AwaitingAsyncTransition);
await TransitionToState(workflowContext, workflowConfiguration.GetFailedStateConfiguration(), null, cancellationToken).ConfigureAwait(false);
return(new WorkflowProcessingResult(workflowInstance, workflowContext.WorkflowExecutionState));
}
workflowContext.WorkflowInstance.CurrentStateProgress = StateExecutionProgress.AfterAsyncTransition;
await ProcessState(workflowContext, stateConfiguration, cancellationToken).ConfigureAwait(false);
}
return(new WorkflowProcessingResult(workflowInstance, workflowContext.WorkflowExecutionState));
}
protected override void WorkflowMessageHandler(IWorkflowMessage message)
{
switch (message.MessageTypeName)
{
default:
Logger.Warning("{0} Did not handle received message [{1}] from [{2}]", ActorId, message.MessageTypeName, Sender.Path);
if (!Sender.IsNobody() && !message.IsReply)
{
Sender.Tell((message as WorkflowMessage).GetWorkflowUnhandledMessage("Message Type Not Implemented", Self.Path));
}
break;
}
}
public async Task Send(string tenantId, IWorkflowMessage message)
{
if (EndpointInstance == null)
{
// If the Endpoint Instance is null, throw an exception so the user gets an error message
throw new Exception($"EndpointInstance is null. {message.ActionType} could not be sent for tenant ID {tenantId}");
}
var options = new SendOptions();
options.SetDestination(MessageQueue);
options.SetHeader(ActionMessageHeaders.TenantId, tenantId);
Log.Info($"Sending Action Message {message.ActionType} for tenant {tenantId}");
var sendTask = EndpointInstance.Send(message, options);
if (await Task.WhenAny(sendTask, Task.Delay(TimeSpan.FromSeconds(SendTimeoutSeconds))) == sendTask)
{
if (sendTask.IsFaulted)
{
var aggregateException = sendTask.Exception;
Log.Error("Send failed for Action Message due to an exception", aggregateException);
var innerException = aggregateException?.InnerException;
if (innerException != null)
{
Log.Error("Inner Exception", innerException);
if (innerException is SqlException)
{
throw new SqlServerSendException(innerException);
}
if (innerException is BrokerUnreachableException)
{
throw new RabbitMqSendException(innerException);
}
throw innerException;
}
if (aggregateException != null)
{
throw aggregateException;
}
throw new Exception("Send failed for Action Message");
}
Log.Info($"Action Message sent successfully for tenant {tenantId}");
}
else
{
var errorMessage = $"Send failed for Action Message due to a timeout after {SendTimeoutSeconds} seconds.";
Log.Error(errorMessage);
throw new Exception(errorMessage);
}
}
/// <summary>
/// Run a workflow step
/// </summary>
/// <param name="workflowMessage">Message to pass to the step class</param>
/// <param name="retryStepTimes">Times to retry on error</param>
/// <param name="workflowStep">The step to run</param>
internal void RunFrameworkStep(IWorkflowMessage workflowMessage, int retryStepTimes, ProcessorStep workflowStep, ProcessorJob currentJob, bool isCheckDepends)
{
bool impersonated = false;
if (!string.IsNullOrEmpty(workflowStep.RunAsDomain) && !string.IsNullOrEmpty(workflowStep.RunAsUser) && !string.IsNullOrEmpty(workflowStep.RunAsPassword) && workflowStep.RunMode == FrameworkStepRunMode.STA)
impersonated = impersonate.ImpersonateValidUser(workflowStep.RunAsUser, workflowStep.RunAsDomain, workflowStep.RunAsPassword);
using (IStep step = StepFactory.GetStep(workflowStep.InvokeClass))
{
try
{
workflowStep.RunStatus = FrameworkStepRunStatus.Loaded;
if (isCheckDepends)
WaitForDependents(workflowStep, currentJob);
workflowStep.StartDate = DateTime.Now;
step.RunStep(workflowMessage);
workflowStep.EndDate = DateTime.Now;
workflowStep.RunStatus = FrameworkStepRunStatus.Complete;
workflowStep.ExitMessage = "Complete";
}
catch (Exception e)
{
workflowStep.RunStatus = FrameworkStepRunStatus.CompleteWithErrors;
workflowStep.ExitMessage = e.Message;
while (e.InnerException != null)
{
e = e.InnerException;
workflowStep.ExitMessage += '|' + e.Message;
}
switch (workflowStep.OnError)
{
case OnFrameworkStepError.RetryStep:
if (workflowStep.WaitBetweenRetriesMilliseconds > 0)
Thread.Sleep(workflowStep.WaitBetweenRetriesMilliseconds);
// Try the step again
retryStepTimes++;
if (retryStepTimes <= workflowStep.RetryTimes)
RunFrameworkStep(workflowMessage, retryStepTimes, workflowStep, currentJob, isCheckDepends);
break;
default:
// If this is running from a thread, do not throw the error up
if (workflowStep.RunMode == FrameworkStepRunMode.STA)
throw e;
else
Processor.ReportJobError(e, workflowStep, workflowMessage, currentJob);
break;
}
}
}
if (impersonated)
impersonate.UndoImpersonation();
}
/// <summary>
/// Runs a framework job
/// </summary>
/// <param name="workflowMessage">The Queue message</param>
/// <param name="retryJobTimes">Times to retry the job or steps</param>
public void RunFrameworkJob(IWorkflowMessage workflowMessage, int retryJobTimes, bool isCheckDepends)
{
// For each job, open a new ProcessorJob to keep track of logging
ProcessorJob currentJob = (ProcessorJob)ProcessorJob.Clone();
foreach (ProcessorStep workflowStep in currentJob.WorkFlowSteps)
{
try
{
int retryStepTimes = 0;
workflowStep.RunStatus = FrameworkStepRunStatus.Waiting;
if (workflowStep.RunMode == FrameworkStepRunMode.STA)
stepExecutor.RunFrameworkStep(workflowMessage, retryStepTimes, workflowStep, currentJob, isCheckDepends);
else if (workflowStep.RunMode == FrameworkStepRunMode.MTA)
Task.Factory.StartNew(() => stepExecutor.RunFrameworkStep(workflowMessage, retryStepTimes, workflowStep, currentJob, isCheckDepends));
}
catch (Exception e)
{
WorkflowException exception = new WorkflowException("Error in framework step " + workflowStep.StepName, e);
workflowStep.ExitMessage = e.Message;
switch (workflowStep.OnError)
{
case OnFrameworkStepError.RetryJob:
if (workflowStep.WaitBetweenRetriesMilliseconds > 0)
Thread.Sleep(workflowStep.WaitBetweenRetriesMilliseconds);
retryJobTimes++;
Processor.ReportJobError(e, workflowStep, workflowMessage, currentJob);
// Try the job again
if (retryJobTimes <= workflowStep.RetryTimes)
RunFrameworkJob(workflowMessage, retryJobTimes, isCheckDepends);
break;
case OnFrameworkStepError.Skip:
// Skip this step. Doing nothing here will skip it
Processor.ReportJobError(e, workflowStep, workflowMessage, currentJob);
break;
case OnFrameworkStepError.Exit:
// Push to to error queue with the error and exit the job
Processor.ReportJobError(e, workflowStep, workflowMessage, currentJob);
return;
}
}
}
// If all steps ran without error report successful job compilation
Processor.ReportJobComplete(workflowMessage, currentJob);
}
/// <summary>
/// The method executed by the framework
/// </summary>
/// <param name="message"></param>
public void RunStep(IWorkflowMessage message)
{
ExampleMessage myMessage = message as ExampleMessage;
if (myMessage == null)
throw new WorkflowStepException("IWorkflowMessage is of the wrong type");
try
{
Parallel.ForEach<string>(Directory.EnumerateFiles(myMessage.CopyFilesFrom, "*"), f =>
{
try
{
File.Copy(f, myMessage.CopyFilesTo + @"\" + Path.GetFileName(f), true);
}catch(Exception){}
});
}
catch (Exception e)
{
throw new WorkflowStepException(e.Message, e);
}
}
public static void HandleComplete(ProcessorQueue processorQueue, IWorkflowMessage message)
{
MessageQueue completedQueue = new System.Messaging.MessageQueue(processorQueue.CompletedQueue);
using (TransactionScope txScope = new TransactionScope(TransactionScopeOption.RequiresNew))
{
try
{
completedQueue.Send(message, MessageQueueTransactionType.Automatic);
txScope.Complete();
}
catch (InvalidOperationException)
{
}
finally
{
completedQueue.Dispose();
}
}
}
/// <summary>
/// The method executed by the framework
/// </summary>
/// <param name="message"></param>
public void RunStep(IWorkflowMessage message)
{
ExampleMessage myMessage = message as ExampleMessage;
if (myMessage == null)
throw new WorkflowStepException("IWorkflowMessage is of the wrong type");
try
{
Parallel.ForEach<string>(Directory.EnumerateFiles(myMessage.CopyFilesTo, "*.jpg"), f =>
{
try
{
File.Move(f, f + "." + Guid.NewGuid());
}
catch (Exception) { }
});
}
catch (Exception e)
{
throw new WorkflowStepException(e.Message, e);
}
}
public async Task SaveWorkflowMessageState(IWorkflowMessage workflowMessage, CancellationToken cancellationToken = default)
{
Guard.ArgumentNotNull(workflowMessage, nameof(workflowMessage));
var mongoCollection = MongoDatabase.GetCollection <WorkflowMessageStateDto>(Constants.WorkflowMessageStateCollectionName);
var existingWorkflowMessageStateDto = mongoCollection.Find(x => x.WorkflowMessageId == workflowMessage.WorkflowMessageId.ToIdString())
.FirstOrDefault();
if (null != existingWorkflowMessageStateDto)
{
throw new Exception($"State for workflow message [{workflowMessage.WorkflowMessageId:D}] has already been saved");
}
var jsonContent = workflowMessage.State.JsonState;
var workflowMessageStateDto = new WorkflowMessageStateDto
{
WorkflowMessageId = workflowMessage.WorkflowMessageId.ToIdString(),
WorkflowInstanceId = workflowMessage.WorkflowInstanceId?.ToIdString(),
Created = DateTime.UtcNow,
JsonState = jsonContent
};
await InsertAsync(mongoCollection, workflowMessageStateDto, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Add a message containing the error to the error queue and the original message to the poison queue
/// </summary>
/// <param name="jobName"></param>
/// <param name="message"></param>
internal static void AddFrameworkError(ProcessorJob processorJob, IWorkflowMessage message, WorkflowErrorMessage errorMessage)
{
try
{
WorkflowConfiguration.LoadFrameworkConfig(processorJob);
ProcessorQueue processorQueue = GetActiveQueue(processorJob, QueueOperationType.Delivery);
MsmqPoisonMessageException error = new MsmqPoisonMessageException() { Source = processorJob.JobName };
error.Data["processorQueue"] = processorQueue;
error.Data["message"] = message;
error.Data["errorMessage"] = errorMessage;
QueueOperationsHandler.HandleError(error);
}
catch (Exception)
{
}
}
/// <summary>
/// Run a new framework job
/// </summary>
/// <param name="workflowMessage"></param>
/// <param name="retryJobTimes"></param>
/// <param name="isCheckDepends"></param>
public void RunFrameworkJob(IWorkflowMessage workflowMessage, int retryJobTimes, bool isCheckDepends)
{
// For each job, open a new ProcessorJob to keep track of logging
ProcessorJob currentJob = (ProcessorJob)processorJob.Clone();
// Add the job for the first pipeline
PipelineInfo pipelineInfo = new PipelineInfo()
{
CurrentJob = currentJob,
IsCheckDepends = isCheckDepends,
RetryJobTimes = retryJobTimes,
RetryStepTimes = 0,
WorkflowMessage = workflowMessage,
IsInProcess = true,
CurrentStepNumber = 0
};
int currentJobsCount = workerBlocks[0].InputCount;
workerBlocks[0].Post(pipelineInfo);
// Block untill a job is finished
// This is done to regulate execution rate
workerBlocks[workerBlocks.Count - 1].OutputAvailableAsync(cts.Token).Wait();
}
/// <summary>
/// Add a job for the framework to process
/// </summary>
/// <param name="jobName">The name of the job in the framework's workflow file</param>
/// <param name="message">A class containing the message data</param>
public static void AddFrameworkJob(string jobName, IWorkflowMessage message)
{
// Add a message to the Queue
ProcessorJob processorJob = new ProcessorJob() { JobName = jobName, CreatedDate = DateTime.Now };
WorkflowConfiguration.LoadFrameworkConfig(processorJob);
ProcessorQueue processorQueue = GetActiveQueue(processorJob, QueueOperationType.Delivery);
MessageQueue workflowQueue = new MessageQueue(processorQueue.MessageQueue);
MessageQueueTransaction transaction = new MessageQueueTransaction();
try
{
if (processorQueue.MessageQueueType == MessageQueueType.Transactional)
{
transaction.Begin();
workflowQueue.Send(message, jobName, transaction);
transaction.Commit();
}
else
{
workflowQueue.Send(message, jobName);
}
}
catch (Exception e)
{
if (processorQueue.MessageQueueType == MessageQueueType.Transactional &&
transaction.Status == MessageQueueTransactionStatus.Pending)
transaction.Abort();
throw new WorkflowException("Error adding message to Queue", e);
}
finally
{
transaction.Dispose();
workflowQueue.Dispose();
}
}
internal static void AddFrameworkJobComplete(ProcessorJob processorJob, IWorkflowMessage message)
{
try
{
WorkflowConfiguration.LoadFrameworkConfig(processorJob);
ProcessorQueue processorQueue = GetActiveQueue(processorJob, QueueOperationType.Delivery);
QueueOperationsHandler.HandleComplete(processorQueue, message);
}
catch (Exception)
{
}
}
public MessageExecutionContext(WorkflowContext workflowContext, IWorkflowMessage workflowMessage)
{
WorkflowContext = workflowContext;
WorkflowMessage = workflowMessage;
}
/// <summary> /// Handle incoming Workflow Messages /// </summary> /// <param name="message"></param> protected abstract void WorkflowMessageHandler(IWorkflowMessage message);
protected void SetWorkflowMessage(IEndpointConfiguration endpointConfiguration, IWorkflowMessage workflowMessage, CancellationToken cancellationToken)
{
if (!WorkflowMessages.ContainsKey(workflowMessage.WorkflowMessageId))
{
WorkflowMessages[workflowMessage.WorkflowMessageId] = workflowMessage;
}
}
/// <summary>
/// Reports a job error
/// </summary>
/// <param name="e">The exception to report</param>
/// <param name="step">The step the exception accrued</param>
/// <param name="workflowMessage">The original message pulled from the queue</param>
public static void ReportJobError(Exception e, ProcessorStep workflowStep, IWorkflowMessage workflowMessage, ProcessorJob currentJob)
{
// Push an error message to the error Queue
WorkflowErrorMessage errorMessage = new WorkflowErrorMessage() { ExceptionMessage = e.Message, JobName = currentJob.JobName, StepName = workflowStep.StepName };
//if e contains inner exceptions, add those messages
while (e.InnerException != null)
{
//assign e to the inner exception - recursive
e = e.InnerException;
errorMessage.ExceptionMessage += '|' + e.Message;
}
FrameworkManager.AddFrameworkError(currentJob, workflowMessage, errorMessage);
}
public async Task SendMessageAsync(string tenantId, IWorkflowMessage message)
{
await _messageTransportHost.SendAsync(tenantId, message);
}
public static void ReportJobComplete(IWorkflowMessage workflowMessage, ProcessorJob currentJob)
{
if (currentJob.NotifyComplete &&
currentJob.WorkFlowSteps.Where(s => s.RunStatus == FrameworkStepRunStatus.Complete).Count() == currentJob.WorkFlowSteps.Count())
FrameworkManager.AddFrameworkJobComplete(currentJob, workflowMessage);
}
public async Task SendAsync(string tenantId, IWorkflowMessage message)
{
Log.Info("Sending message to server.");
await _nServiceBusServer.Send(tenantId, message);
}
